web: add web-app server for development and production builds (#20126)

This PR introduces an independent server for the web application, which can use any deployed API instance as a backend.

Author: valerybugakov

client/web/README.md

@@ -0,0 +1,47 @@
+# Web Application
+
+## Local development
+
+Use `sg` CLI tool to configure and start local development server. For more information checkout `sg` [README]('../../dev/sg/README.md').
+
+### Configuration
+
+Environment variables important for the web server:
+
+1. `WEBPACK_SERVE_INDEX` should be set to `true` to enable `HTMLWebpackPlugin`.
+2. `SOURCEGRAPH_API_URL` is used as a proxied API url. By default it points to the [https://k8s.sgdev.org](https://k8s.sgdev.org).
+
+It's possible to overwrite these variables by creating `sg.config.overwrite.yaml` in the root folder and adjusting the `env` section of the relevant command.
+
+### Development server
+
+```sh
+sg run web-standalone
+```
+
+For enterprise version:
+
+```sh
+sg run enterprise-web-standalone
+```
+
+### Production server
+
+```sh
+sg run web-standalone-prod
+```
+
+For enterprise version:
+
+```sh
+sg run enterprise-web-standalone-prod
+```
+
+Web app should be available at `http://${SOURCEGRAPH_HTTPS_DOMAIN}:${SOURCEGRAPH_HTTPS_PORT}`.
+Build artifacts will be served from `<rootRepoPath>/ui/assets`.
+
+### API proxy
+
+In both environments, server proxies API requests to `SOURCEGRAPH_API_URL` provided as the `.env` variable.
+To avoid the `CSRF token is invalid` error CSRF token is retrieved from the `SOURCEGRAPH_API_URL` before the server starts.
+Then this value is used for every subsequent request to the API.

client/web/dev/server/development.server.ts

@@ -0,0 +1,63 @@
+import chalk from 'chalk'
+import signale from 'signale'
+import createWebpackCompiler, { Configuration } from 'webpack'
+import WebpackDevServer, { ProxyConfigArrayItem } from 'webpack-dev-server'
+
+import {
+    getCSRFTokenCookieMiddleware,
+    PROXY_ROUTES,
+    environmentConfig,
+    getAPIProxySettings,
+    getCSRFTokenAndCookie,
+    STATIC_ASSETS_PATH,
+    STATIC_ASSETS_URL,
+    WEBPACK_STATS_OPTIONS,
+    WEB_SERVER_URL,
+} from '../utils'
+
+// TODO: migrate webpack.config.js to TS to use `import` in this file.
+// eslint-disable-next-line @typescript-eslint/no-var-requires, @typescript-eslint/no-require-imports
+const webpackConfig = require('../../webpack.config') as Configuration
+const { SOURCEGRAPH_API_URL, SOURCEGRAPH_HTTPS_PORT, IS_HOT_RELOAD_ENABLED } = environmentConfig
+
+export async function startDevelopmentServer(): Promise<void> {
+    // Get CSRF token value from the `SOURCEGRAPH_API_URL`.
+    const { csrfContextValue, csrfCookieValue } = await getCSRFTokenAndCookie(SOURCEGRAPH_API_URL)
+    signale.await('Development server', { ...environmentConfig, csrfContextValue, csrfCookieValue })
+
+    const proxyConfig = {
+        context: PROXY_ROUTES,
+        ...getAPIProxySettings({
+            csrfContextValue,
+            apiURL: SOURCEGRAPH_API_URL,
+        }),
+    }
+
+    const options: WebpackDevServer.Configuration = {
+        hot: IS_HOT_RELOAD_ENABLED,
+        // TODO: resolve https://github.com/webpack/webpack-dev-server/issues/2313 and enable HTTPS.
+        https: false,
+        historyApiFallback: true,
+        port: SOURCEGRAPH_HTTPS_PORT,
+        publicPath: STATIC_ASSETS_URL,
+        contentBase: STATIC_ASSETS_PATH,
+        contentBasePublicPath: [STATIC_ASSETS_URL, '/'],
+        stats: WEBPACK_STATS_OPTIONS,
+        noInfo: false,
+        disableHostCheck: true,
+        proxy: [proxyConfig as ProxyConfigArrayItem],
+        before(app) {
+            app.use(getCSRFTokenCookieMiddleware(csrfCookieValue))
+        },
+    }
+
+    WebpackDevServer.addDevServerEntrypoints(webpackConfig, options)
+
+    const server = new WebpackDevServer(createWebpackCompiler(webpackConfig), options)
+
+    server.listen(SOURCEGRAPH_HTTPS_PORT, '0.0.0.0', () => {
+        signale.success(`Development server is ready at ${chalk.blue.bold(WEB_SERVER_URL)}`)
+    })
+}
+
+startDevelopmentServer().catch(error => signale.error(error))

client/web/dev/server/production.server.ts

@@ -0,0 +1,53 @@
+import chalk from 'chalk'
+import historyApiFallback from 'connect-history-api-fallback'
+import express, { RequestHandler } from 'express'
+import { createProxyMiddleware } from 'http-proxy-middleware'
+import signale from 'signale'
+
+import {
+    PROXY_ROUTES,
+    getAPIProxySettings,
+    getCSRFTokenCookieMiddleware,
+    environmentConfig,
+    getCSRFTokenAndCookie,
+    STATIC_ASSETS_PATH,
+    WEB_SERVER_URL,
+} from '../utils'
+
+const { SOURCEGRAPH_API_URL, SOURCEGRAPH_HTTPS_PORT } = environmentConfig
+
+async function startProductionServer(): Promise<void> {
+    // Get CSRF token value from the `SOURCEGRAPH_API_URL`.
+    const { csrfContextValue, csrfCookieValue } = await getCSRFTokenAndCookie(SOURCEGRAPH_API_URL)
+    signale.await('Production server', { ...environmentConfig, csrfContextValue, csrfCookieValue })
+
+    const app = express()
+
+    // Serve index.html in place of any 404 responses.
+    app.use(historyApiFallback() as RequestHandler)
+    // Attach `CSRF_COOKIE_NAME` cookie to every response to avoid "CSRF token is invalid" API error.
+    app.use(getCSRFTokenCookieMiddleware(csrfCookieValue))
+
+    // Serve index.html.
+    app.use(express.static(STATIC_ASSETS_PATH))
+    // Serve build artifacts.
+    app.use('/.assets', express.static(STATIC_ASSETS_PATH))
+
+    // Proxy API requests to the `process.env.SOURCEGRAPH_API_URL`.
+    app.use(
+        PROXY_ROUTES,
+        createProxyMiddleware(
+            getAPIProxySettings({
+                // Attach `x-csrf-token` header to every proxy request.
+                csrfContextValue,
+                apiURL: SOURCEGRAPH_API_URL,
+            })
+        )
+    )
+
+    app.listen(SOURCEGRAPH_HTTPS_PORT, () => {
+        signale.success(`Production server is ready at ${chalk.blue.bold(WEB_SERVER_URL)}`)
+    })
+}
+
+startProductionServer().catch(error => signale.error(error))

client/web/dev/tsconfig.json

@@ -0,0 +1,6 @@
+{
+  "extends": "../tsconfig.json",
+  "compilerOptions": {
+    "module": "commonjs",
+  },
+}

client/web/dev/utils/constants.ts

@@ -0,0 +1,14 @@
+import path from 'path'
+
+export const ROOT_PATH = path.resolve(__dirname, '../../../../')
+export const STATIC_ASSETS_PATH = path.resolve(ROOT_PATH, 'ui/assets')
+export const STATIC_ASSETS_URL = '/.assets/'
+
+// TODO: share with gulpfile.js
+export const WEBPACK_STATS_OPTIONS = {
+    all: false,
+    timings: true,
+    errors: true,
+    warnings: true,
+    colors: true,
+}

client/web/dev/utils/create-js-context.ts

@@ -0,0 +1,56 @@
+import { SourcegraphContext } from '../../src/jscontext'
+
+import { getSiteConfig } from './get-site-config'
+
+// TODO: share with `client/web/src/integration/jscontext` which is not included into `tsconfig.json` now.
+export const builtinAuthProvider = {
+    serviceType: 'builtin' as const,
+    serviceID: '',
+    clientID: '',
+    displayName: 'Builtin username-password authentication',
+    isBuiltin: true,
+    authenticationURL: '',
+}
+
+// Create dummy JS context that will be added to index.html when `WEBPACK_SERVE_INDEX` is set to true.
+export const createJsContext = ({ sourcegraphBaseUrl }: { sourcegraphBaseUrl: string }): SourcegraphContext => {
+    const siteConfig = getSiteConfig()
+
+    if (siteConfig?.authProviders) {
+        siteConfig.authProviders.unshift(builtinAuthProvider)
+    }
+
+    return {
+        externalURL: sourcegraphBaseUrl,
+        accessTokensAllow: 'all-users-create',
+        allowSignup: true,
+        batchChangesEnabled: true,
+        codeIntelAutoIndexingEnabled: false,
+        externalServicesUserModeEnabled: true,
+        productResearchPageEnabled: true,
+        csrfToken: 'qwerty',
+        assetsRoot: '/.assets',
+        deployType: 'dev',
+        debug: true,
+        emailEnabled: false,
+        experimentalFeatures: {},
+        isAuthenticatedUser: true,
+        likelyDockerOnMac: false,
+        needServerRestart: false,
+        needsSiteInit: false,
+        resetPasswordEnabled: true,
+        sentryDSN: null,
+        site: {
+            'update.channel': 'release',
+        },
+        siteID: 'TestSiteID',
+        siteGQLID: 'TestGQLSiteID',
+        sourcegraphDotComMode: true,
+        userAgentIsBot: false,
+        version: '0.0.0',
+        xhrHeaders: {},
+        authProviders: [builtinAuthProvider],
+        // Site-config overrides default JS context
+        ...siteConfig,
+    }
+}

client/web/dev/utils/csrf/get-csrf-token-and-cookie.ts

@@ -0,0 +1,43 @@
+import fetch from 'node-fetch'
+
+export const CSRF_CONTEXT_KEY = 'csrfToken'
+const CSRF_CONTEXT_VALUE_REGEXP = new RegExp(`${CSRF_CONTEXT_KEY}":"(.*?)"`)
+
+export const CSRF_COOKIE_NAME = 'sg_csrf_token'
+const CSRF_COOKIE_VALUE_REGEXP = new RegExp(`${CSRF_COOKIE_NAME}=(.*?);`)
+
+interface CSFRTokenAndCookie {
+    csrfContextValue: string
+    csrfCookieValue: string
+}
+
+/**
+ *
+ * Fetch `${proxyUrl}/sign-in` and extract two values from the response:
+ *
+ * 1. `set-cookie` value for `CSRF_COOKIE_NAME`.
+ * 2. value from JS context under `CSRF_CONTEXT_KEY` key.
+ *
+ */
+export async function getCSRFTokenAndCookie(proxyUrl: string): Promise<CSFRTokenAndCookie> {
+    const response = await fetch(`${proxyUrl}/sign-in`)
+
+    const html = await response.text()
+    const cookieHeader = response.headers.get('set-cookie')
+
+    if (!cookieHeader) {
+        throw new Error(`"set-cookie" header not found in "${proxyUrl}/sign-in" response`)
+    }
+
+    const csrfHeaderMatches = CSRF_CONTEXT_VALUE_REGEXP.exec(html)
+    const csrfCookieMatches = CSRF_COOKIE_VALUE_REGEXP.exec(cookieHeader)
+
+    if (!csrfHeaderMatches || !csrfCookieMatches) {
+        throw new Error('CSRF value not found!')
+    }
+
+    return {
+        csrfContextValue: csrfHeaderMatches[1],
+        csrfCookieValue: csrfCookieMatches[1],
+    }
+}

client/web/dev/utils/csrf/get-csrf-token-cookie-middleware.ts

@@ -0,0 +1,9 @@
+import { RequestHandler } from 'express'
+
+import { CSRF_COOKIE_NAME } from './get-csrf-token-and-cookie'
+
+// Attach `CSRF_COOKIE_NAME` cookie to every response to avoid "CSRF token is invalid" API error.
+export const getCSRFTokenCookieMiddleware = (csrfCookieValue: string): RequestHandler => (_request, response, next) => {
+    response.cookie(CSRF_COOKIE_NAME, csrfCookieValue, { httpOnly: true })
+    next()
+}

client/web/dev/utils/csrf/index.ts

@@ -0,0 +1,2 @@
+export * from './get-csrf-token-and-cookie'
+export * from './get-csrf-token-cookie-middleware'

client/web/dev/utils/environment-config.ts

@@ -0,0 +1,22 @@
+import path from 'path'
+
+import { ROOT_PATH } from './constants'
+
+const DEFAULT_SITE_CONFIG_PATH = path.resolve(ROOT_PATH, '../dev-private/enterprise/dev/site-config.json')
+
+export const environmentConfig = {
+    NODE_ENV: process.env.NODE_ENV || 'development',
+    SOURCEGRAPH_API_URL: process.env.SOURCEGRAPH_API_URL || 'https://k8s.sgdev.org',
+    SOURCEGRAPH_HTTPS_DOMAIN: process.env.SOURCEGRAPH_HTTPS_DOMAIN || 'sourcegraph.test',
+    SOURCEGRAPH_HTTPS_PORT: Number(process.env.SOURCEGRAPH_HTTPS_PORT) || 3443,
+    WEBPACK_SERVE_INDEX: process.env.WEBPACK_SERVE_INDEX === 'true',
+    SITE_CONFIG_PATH: process.env.SITE_CONFIG_PATH || DEFAULT_SITE_CONFIG_PATH,
+    ENTERPRISE: Boolean(process.env.ENTERPRISE),
+
+    // TODO: do we use process.env.NO_HOT anywhere?
+    IS_HOT_RELOAD_ENABLED: process.env.NO_HOT !== 'true',
+}
+
+const { SOURCEGRAPH_HTTPS_PORT, SOURCEGRAPH_HTTPS_DOMAIN } = environmentConfig
+
+export const WEB_SERVER_URL = `http://${SOURCEGRAPH_HTTPS_DOMAIN}:${SOURCEGRAPH_HTTPS_PORT}`

client/web/dev/utils/get-api-proxy-settings.ts

@@ -0,0 +1,42 @@
+import { Options } from 'http-proxy-middleware'
+
+// One of the API routes: "/-/sign-in".
+export const PROXY_ROUTES = ['/.api', '/search/stream', '/-', '/.auth']
+
+interface GetAPIProxySettingsOptions {
+    csrfContextValue: string
+    apiURL: string
+}
+
+export const getAPIProxySettings = ({ csrfContextValue, apiURL }: GetAPIProxySettingsOptions): Options => ({
+    target: apiURL,
+    // Do not SSL certificate.
+    secure: false,
+    // Change the origin of the host header to the target URL.
+    changeOrigin: true,
+    // Attach `x-csrf-token` header to every request to avoid "CSRF token is invalid" API error.
+    headers: {
+        'x-csrf-token': csrfContextValue,
+    },
+    // Rewrite domain of `set-cookie` headers for all cookies received.
+    cookieDomainRewrite: '',
+    onProxyRes: proxyResponse => {
+        if (proxyResponse.headers['set-cookie']) {
+            // Remove `Secure` and `SameSite` from `set-cookie` headers.
+            const cookies = proxyResponse.headers['set-cookie'].map(cookie =>
+                cookie.replace(/; secure/gi, '').replace(/; samesite=.+/gi, '')
+            )
+
+            proxyResponse.headers['set-cookie'] = cookies
+        }
+    },
+    // TODO: share with `client/web/gulpfile.js`
+    // Avoid crashing on "read ECONNRESET".
+    onError: () => undefined,
+    // Don't log proxy errors, these usually just contain
+    // ECONNRESET errors caused by the browser cancelling
+    // requests. This should not be needed to actually debug something.
+    logLevel: 'silent',
+    onProxyReqWs: (_proxyRequest, _request, socket) =>
+        socket.on('error', error => console.error('WebSocket proxy error:', error)),
+})

client/web/dev/utils/get-site-config.ts

@@ -0,0 +1,25 @@
+import fs from 'fs'
+
+import { parse } from '@sqs/jsonc-parser'
+import lodash from 'lodash'
+
+import { SourcegraphContext } from '../../src/jscontext'
+
+import { environmentConfig } from './environment-config'
+
+const { SITE_CONFIG_PATH } = environmentConfig
+
+// Get site-config from `SITE_CONFIG_PATH` as an object with camel cased keys.
+export const getSiteConfig = (): Partial<SourcegraphContext> => {
+    try {
+        // eslint-disable-next-line no-sync
+        const siteConfig = parse(fs.readFileSync(SITE_CONFIG_PATH, 'utf-8'))
+
+        return lodash.mapKeys(siteConfig, (_value, key) => lodash.camelCase(key))
+    } catch (error) {
+        console.log('Site config not found!', SITE_CONFIG_PATH)
+        console.error(error)
+
+        return {}
+    }
+}

client/web/dev/utils/index.ts

@@ -0,0 +1,5 @@
+export * from './constants'
+export * from './create-js-context'
+export * from './environment-config'
+export * from './get-api-proxy-settings'
+export * from './csrf'