feat: added support for customUserAgent (#2825)

src/core/client/admin/routes/Configure/sections/Advanced/StoryCreationConfig.tsx

@@ -26,6 +26,7 @@ graphql`
       scraping {
         enabled
         proxyURL
+        customUserAgent
       }
       disableLazy
     }
@@ -117,6 +118,39 @@ const StoryCreationConfig: FunctionComponent<Props> = ({ disabled }) => (
         )}
       </Field>
     </FormField>
+    <FormField>
+      <FormFieldHeader>
+        <Localized id="configure-advanced-stories-custom-user-agent">
+          <Label htmlFor="configure-advanced-stories-custom-user-agent">
+            Custom Scraper User Agent Header
+          </Label>
+        </Localized>
+        <Localized
+          id="configure-advanced-stories-custom-user-agent-detail"
+          code={<code />}
+        >
+          <HelperText>
+            When specified, overrides the <code>User-Agent</code> header sent
+            with each scrape request.
+          </HelperText>
+        </Localized>
+      </FormFieldHeader>
+      <Field name="stories.scraping.customUserAgent" parse={parseEmptyAsNull}>
+        {({ input, meta }) => (
+          <TextFieldWithValidation
+            {...input}
+            id="configure-advanced-stories-custom-user-agent"
+            disabled={disabled}
+            fullWidth
+            autoComplete="off"
+            autoCorrect="off"
+            autoCapitalize="off"
+            spellCheck={false}
+            meta={meta}
+          />
+        )}
+      </Field>
+    </FormField>
   </ConfigBox>
 );
 

src/core/client/admin/test/configure/__snapshots__/advanced.spec.tsx.snap

@@ -625,6 +625,54 @@ proxy as parsed by the
                     </div>
                   </div>
                 </div>
+                <div
+                  className="Box-root HorizontalGutter-root FormField-root HorizontalGutter-spacing-2"
+                >
+                  <div
+                    className="Box-root HorizontalGutter-root FormFieldHeader-root HorizontalGutter-spacing-1"
+                  >
+                    <label
+                      className="Label-root"
+                      htmlFor="configure-advanced-stories-custom-user-agent"
+                    >
+                      Custom Scraper User Agent Header
+                    </label>
+                    <p
+                      className="HelperText-root"
+                    >
+                      When specified, overrides the 
+                      <code>
+                        User-Agent
+                      </code>
+                       header sent with each
+scrape request.
+                    </p>
+                  </div>
+                  <div
+                    className="Box-root HorizontalGutter-root HorizontalGutter-spacing-2"
+                  >
+                    <div
+                      className="TextField-root TextField-fullWidth"
+                    >
+                      <input
+                        autoCapitalize="off"
+                        autoComplete="off"
+                        autoCorrect="off"
+                        className="TextField-input TextField-colorRegular"
+                        disabled={false}
+                        id="configure-advanced-stories-custom-user-agent"
+                        name="stories.scraping.customUserAgent"
+                        onBlur={[Function]}
+                        onChange={[Function]}
+                        onFocus={[Function]}
+                        placeholder=""
+                        spellCheck={false}
+                        type="text"
+                        value=""
+                      />
+                    </div>
+                  </div>
+                </div>
               </div>
             </div>
           </div>

src/core/server/graph/loaders/Stories.ts

@@ -119,9 +119,12 @@ export default (ctx: GraphContext) => ({
       // This typecast is needed because the custom `ms` format does not return
       // the desired `number` type even though that's the only type it can
       // output.
-      scraper.scrape(url, (ctx.config.get(
-        "scrape_timeout"
-      ) as unknown) as number)
+      scraper.scrape(
+        url,
+        (ctx.config.get("scrape_timeout") as unknown) as number,
+        ctx.tenant.stories.scraping.customUserAgent,
+        ctx.tenant.stories.scraping.proxyURL
+      )
     ),
     {
       // Disable caching for the DataLoader if the Context is designed to be

src/core/server/graph/schema/schema.graphql

@@ -1150,6 +1150,12 @@ type StoryScrapingConfiguration {
   the [proxy-agent](https://www.npmjs.com/package/proxy-agent) package.
   """
   proxyURL: String
+
+  """
+  customUserAgent when specified will override the user agent used by fetch
+  requests made during the scraping process.
+  """
+  customUserAgent: String
 }
 
 """
@@ -3519,6 +3525,12 @@ input StoryScrapingConfigurationInput {
   the [proxy-agent](https://www.npmjs.com/package/proxy-agent) package.
   """
   proxyURL: String
+
+  """
+  customUserAgent when specified will override the user agent used by fetch
+  requests made during the scraping process.
+  """
+  customUserAgent: String
 }
 
 """

src/core/server/services/stories/scraper/scraper.ts

@@ -86,6 +86,7 @@ class Scraper {
   public async download(
     url: string,
     abortAfterMilliseconds: number,
+    customUserAgent?: string,
     proxyURL?: string
   ) {
     const log = this.log.child({ storyURL: url }, true);
@@ -95,7 +96,7 @@ class Scraper {
 
     const options: RequestInit = {
       headers: {
-        "User-Agent": `Talk Scraper/${version}`,
+        "User-Agent": customUserAgent || `Talk Scraper/${version}`,
       },
       signal: controller.signal,
     };
@@ -136,9 +137,15 @@ class Scraper {
   public async scrape(
     url: string,
     abortAfterMilliseconds: number,
+    customUserAgent?: string,
     proxyURL?: string
   ): Promise<GQLStoryMetadata | null> {
-    const html = await this.download(url, abortAfterMilliseconds, proxyURL);
+    const html = await this.download(
+      url,
+      abortAfterMilliseconds,
+      customUserAgent,
+      proxyURL
+    );
     if (!html) {
       return null;
     }
@@ -199,6 +206,7 @@ export async function scrape(
   const metadata = await scraper.scrape(
     storyURL,
     abortAfterMilliseconds,
+    tenant.stories.scraping.customUserAgent,
     tenant.stories.scraping.proxyURL
   );
   if (!metadata) {

src/locales/en-US/admin.ftl

@@ -885,6 +885,10 @@ configure-advanced-stories-proxy-detail =
   When specified, allows scraping requests to use the provided
   proxy. All requests will then be passed through the appropriote
   proxy as parsed by the <externalLink>npm proxy-agent</externalLink> package.
+configure-advanced-stories-custom-user-agent = Custom Scraper User Agent Header
+configure-advanced-stories-custom-user-agent-detail =
+  When specified, overrides the <code>User-Agent</code> header sent with each
+  scrape request.
 
 forgotPassword-forgotPasswordHeader = Forgot password?
 forgotPassword-checkEmailHeader = Check your email