feat: support csp (#9111)

Co-authored-by: Joel <iamjoel007@gmail.com>
2024-11-16 03:32:23 +08:00 · 2024-10-11 16:14:56 +08:00 · 2024-10-11 16:14:56 +08:00 · 0c3d3c3993
commit 0c3d3c3993
parent 409676d0ce
10 changed files with 148 additions and 64 deletions
--- a/docker/.env.example
+++ b/docker/.env.example
@ -797,4 +797,6 @@ POSITION_TOOL_EXCLUDES=
 # Example: POSITION_PROVIDER_PINS=openai,openllm
 POSITION_PROVIDER_PINS=
 POSITION_PROVIDER_INCLUDES=
-POSITION_PROVIDER_EXCLUDES=
+POSITION_PROVIDER_EXCLUDES=
+# CSP https://developer.mozilla.org/en-US/docs/Web/HTTP/CSP
+CSP_WHITELIST=
--- a/docker/docker-compose.yaml
+++ b/docker/docker-compose.yaml
@ -261,6 +261,7 @@ services:
      SENTRY_DSN: ${WEB_SENTRY_DSN:-}
      NEXT_TELEMETRY_DISABLED: ${NEXT_TELEMETRY_DISABLED:-0}
      TEXT_GENERATION_TIMEOUT_MS: ${TEXT_GENERATION_TIMEOUT_MS:-60000}
+      CSP_WHITELIST: ${CSP_WHITELIST:-}

  # The postgres database.
  db:
@ -280,7 +281,7 @@ services:
    volumes:
      - ./volumes/db/data:/var/lib/postgresql/data
    healthcheck:
-      test: [ "CMD", "pg_isready" ]
+      test: ['CMD', 'pg_isready']
      interval: 1s
      timeout: 3s
      retries: 30
@ -295,7 +296,7 @@ services:
    # Set the redis password when startup redis server.
    command: redis-server --requirepass ${REDIS_PASSWORD:-difyai123456}
    healthcheck:
-      test: [ "CMD", "redis-cli", "ping" ]
+      test: ['CMD', 'redis-cli', 'ping']

  # The DifySandbox
  sandbox:
@ -315,7 +316,7 @@ services:
    volumes:
      - ./volumes/sandbox/dependencies:/dependencies
    healthcheck:
-      test: [ "CMD", "curl", "-f", "http://localhost:8194/health" ]
+      test: ['CMD', 'curl', '-f', 'http://localhost:8194/health']
    networks:
      - ssrf_proxy_network

@ -328,7 +329,12 @@ services:
    volumes:
      - ./ssrf_proxy/squid.conf.template:/etc/squid/squid.conf.template
      - ./ssrf_proxy/docker-entrypoint.sh:/docker-entrypoint-mount.sh
-    entrypoint: [ "sh", "-c", "cp /docker-entrypoint-mount.sh /docker-entrypoint.sh && sed -i 's/\r$$//' /docker-entrypoint.sh && chmod +x /docker-entrypoint.sh && /docker-entrypoint.sh" ]
+    entrypoint:
+      [
+        'sh',
+        '-c',
+        "cp /docker-entrypoint-mount.sh /docker-entrypoint.sh && sed -i 's/\r$$//' /docker-entrypoint.sh && chmod +x /docker-entrypoint.sh && /docker-entrypoint.sh",
+      ]
    environment:
      # pls clearly modify the squid env vars to fit your network environment.
      HTTP_PORT: ${SSRF_HTTP_PORT:-3128}
@ -357,8 +363,8 @@ services:
      - CERTBOT_EMAIL=${CERTBOT_EMAIL}
      - CERTBOT_DOMAIN=${CERTBOT_DOMAIN}
      - CERTBOT_OPTIONS=${CERTBOT_OPTIONS:-}
-    entrypoint: [ "/docker-entrypoint.sh" ]
-    command: [ "tail", "-f", "/dev/null" ]
+    entrypoint: ['/docker-entrypoint.sh']
+    command: ['tail', '-f', '/dev/null']

  # The nginx reverse proxy.
  # used for reverse proxying the API service and Web service.
@ -375,7 +381,12 @@ services:
      - ./volumes/certbot/conf/live:/etc/letsencrypt/live # cert dir (with certbot container)
      - ./volumes/certbot/conf:/etc/letsencrypt
      - ./volumes/certbot/www:/var/www/html
-    entrypoint: [ "sh", "-c", "cp /docker-entrypoint-mount.sh /docker-entrypoint.sh && sed -i 's/\r$$//' /docker-entrypoint.sh && chmod +x /docker-entrypoint.sh && /docker-entrypoint.sh" ]
+    entrypoint:
+      [
+        'sh',
+        '-c',
+        "cp /docker-entrypoint-mount.sh /docker-entrypoint.sh && sed -i 's/\r$$//' /docker-entrypoint.sh && chmod +x /docker-entrypoint.sh && /docker-entrypoint.sh",
+      ]
    environment:
      NGINX_SERVER_NAME: ${NGINX_SERVER_NAME:-_}
      NGINX_HTTPS_ENABLED: ${NGINX_HTTPS_ENABLED:-false}
@ -397,14 +408,14 @@ services:
      - api
      - web
    ports:
-      - "${EXPOSE_NGINX_PORT:-80}:${NGINX_PORT:-80}"
-      - "${EXPOSE_NGINX_SSL_PORT:-443}:${NGINX_SSL_PORT:-443}"
+      - '${EXPOSE_NGINX_PORT:-80}:${NGINX_PORT:-80}'
+      - '${EXPOSE_NGINX_SSL_PORT:-443}:${NGINX_SSL_PORT:-443}'

  # The Weaviate vector store.
  weaviate:
    image: semitechnologies/weaviate:1.19.0
    profiles:
-      - ""
+      - ''
      - weaviate
    restart: always
    volumes:
@ -453,7 +464,7 @@ services:
    volumes:
      - ./volumes/pgvector/data:/var/lib/postgresql/data
    healthcheck:
-      test: [ "CMD", "pg_isready" ]
+      test: ['CMD', 'pg_isready']
      interval: 1s
      timeout: 3s
      retries: 30
@ -475,7 +486,7 @@ services:
    volumes:
      - ./volumes/pgvecto_rs/data:/var/lib/postgresql/data
    healthcheck:
-      test: [ "CMD", "pg_isready" ]
+      test: ['CMD', 'pg_isready']
      interval: 1s
      timeout: 3s
      retries: 30
@ -523,7 +534,7 @@ services:
      - ./volumes/milvus/etcd:/etcd
    command: etcd -advertise-client-urls=http://127.0.0.1:2379 -listen-client-urls http://0.0.0.0:2379 --data-dir /etcd
    healthcheck:
-      test: [ "CMD", "etcdctl", "endpoint", "health" ]
+      test: ['CMD', 'etcdctl', 'endpoint', 'health']
      interval: 30s
      timeout: 20s
      retries: 3
@ -542,7 +553,7 @@ services:
      - ./volumes/milvus/minio:/minio_data
    command: minio server /minio_data --console-address ":9001"
    healthcheck:
-      test: [ "CMD", "curl", "-f", "http://localhost:9000/minio/health/live" ]
+      test: ['CMD', 'curl', '-f', 'http://localhost:9000/minio/health/live']
      interval: 30s
      timeout: 20s
      retries: 3
@ -554,7 +565,7 @@ services:
    image: milvusdb/milvus:v2.3.1
    profiles:
      - milvus
-    command: [ "milvus", "run", "standalone" ]
+    command: ['milvus', 'run', 'standalone']
    environment:
      ETCD_ENDPOINTS: ${ETCD_ENDPOINTS:-etcd:2379}
      MINIO_ADDRESS: ${MINIO_ADDRESS:-minio:9000}
@ -562,7 +573,7 @@ services:
    volumes:
      - ./volumes/milvus/milvus:/var/lib/milvus
    healthcheck:
-      test: [ "CMD", "curl", "-f", "http://localhost:9091/healthz" ]
+      test: ['CMD', 'curl', '-f', 'http://localhost:9091/healthz']
      interval: 30s
      start_period: 90s
      timeout: 20s
@ -644,13 +655,13 @@ services:
      node.name: dify-es0
      discovery.type: single-node
      xpack.license.self_generated.type: trial
-      xpack.security.enabled: "true"
-      xpack.security.enrollment.enabled: "false"
-      xpack.security.http.ssl.enabled: "false"
+      xpack.security.enabled: 'true'
+      xpack.security.enrollment.enabled: 'false'
+      xpack.security.http.ssl.enabled: 'false'
    ports:
      - ${ELASTICSEARCH_PORT:-9200}:9200
    healthcheck:
-      test: [ "CMD", "curl", "-s", "http://localhost:9200/_cluster/health?pretty" ]
+      test: ['CMD', 'curl', '-s', 'http://localhost:9200/_cluster/health?pretty']
      interval: 30s
      timeout: 10s
      retries: 50
@ -668,17 +679,17 @@ services:
    environment:
      XPACK_ENCRYPTEDSAVEDOBJECTS_ENCRYPTIONKEY: d1a66dfd-c4d3-4a0a-8290-2abcb83ab3aa
      NO_PROXY: localhost,127.0.0.1,elasticsearch,kibana
-      XPACK_SECURITY_ENABLED: "true"
-      XPACK_SECURITY_ENROLLMENT_ENABLED: "false"
-      XPACK_SECURITY_HTTP_SSL_ENABLED: "false"
-      XPACK_FLEET_ISAIRGAPPED: "true"
+      XPACK_SECURITY_ENABLED: 'true'
+      XPACK_SECURITY_ENROLLMENT_ENABLED: 'false'
+      XPACK_SECURITY_HTTP_SSL_ENABLED: 'false'
+      XPACK_FLEET_ISAIRGAPPED: 'true'
      I18N_LOCALE: zh-CN
-      SERVER_PORT: "5601"
+      SERVER_PORT: '5601'
      ELASTICSEARCH_HOSTS: http://elasticsearch:9200
    ports:
      - ${KIBANA_PORT:-5601}:5601
    healthcheck:
-      test: [ "CMD-SHELL", "curl -s http://localhost:5601 >/dev/null || exit 1" ]
+      test: ['CMD-SHELL', 'curl -s http://localhost:5601 >/dev/null || exit 1']
      interval: 30s
      timeout: 10s
      retries: 3
--- a/web/.env.example
+++ b/web/.env.example
@ -22,3 +22,6 @@ NEXT_PUBLIC_UPLOAD_IMAGE_AS_ICON=false

 # The timeout for the text generation in millisecond
 NEXT_PUBLIC_TEXT_GENERATION_TIMEOUT_MS=60000
+
+# CSP https://developer.mozilla.org/en-US/docs/Web/HTTP/CSP
+NEXT_PUBLIC_CSP_WHITELIST=
--- a/web/app/components/base/ga/index.tsx
+++ b/web/app/components/base/ga/index.tsx
@ -1,6 +1,7 @@
 import type { FC } from 'react'
 import React from 'react'
 import Script from 'next/script'
+import { headers } from 'next/headers'
 import { IS_CE_EDITION } from '@/config'

 export enum GaType {
@ -23,9 +24,16 @@ const GA: FC<IGAProps> = ({
  if (IS_CE_EDITION)
    return null

+  const nonce = process.env.NODE_ENV === 'production' ? headers().get('x-nonce') : ''
+
  return (
    <>
-      <Script strategy="beforeInteractive" async src={`https://www.googletagmanager.com/gtag/js?id=${gaIdMaps[gaType]}`}></Script>
+      <Script
+        strategy="beforeInteractive"
+        async
+        src={`https://www.googletagmanager.com/gtag/js?id=${gaIdMaps[gaType]}`}
+        nonce={nonce!}
+      ></Script>
      <Script
        id="ga-init"
        dangerouslySetInnerHTML={{
@ -36,6 +44,7 @@ gtag('js', new Date());
 gtag('config', '${gaIdMaps[gaType]}');
          `,
        }}
+        nonce={nonce!}
      >
      </Script>
    </>
--- a/web/app/components/base/topbar/index.tsx
+++ b/web/app/components/base/topbar/index.tsx
@ -1,16 +0,0 @@
-'use client'
-
-import { AppProgressBar as ProgressBar } from 'next-nprogress-bar'
-
-const Topbar = () => {
-  return (
-    <>
-      <ProgressBar
-        height='2px'
-        color="#1C64F2FF"
-        options={{ showSpinner: false }}
-        shallowRouting />
-    </>)
-}
-
-export default Topbar
--- a/web/app/layout.tsx
+++ b/web/app/layout.tsx
@ -2,7 +2,6 @@ import type { Viewport } from 'next'
 import I18nServer from './components/i18n-server'
 import BrowserInitor from './components/browser-initor'
 import SentryInitor from './components/sentry-initor'
-import Topbar from './components/base/topbar'
 import { getLocaleOnServer } from '@/i18n/server'
 import './styles/globals.css'
 import './styles/markdown.scss'
@ -45,7 +44,6 @@ const LocaleLayout = ({
        data-public-site-about={process.env.NEXT_PUBLIC_SITE_ABOUT}
        data-public-text-generation-timeout-ms={process.env.NEXT_PUBLIC_TEXT_GENERATION_TIMEOUT_MS}
      >
-        <Topbar />
        <BrowserInitor>
          <SentryInitor>
            <I18nServer>{children}</I18nServer>
--- a/web/docker/entrypoint.sh
+++ b/web/docker/entrypoint.sh
@ -22,5 +22,6 @@ export NEXT_PUBLIC_SITE_ABOUT=${SITE_ABOUT}
 export NEXT_TELEMETRY_DISABLED=${NEXT_TELEMETRY_DISABLED}

 export NEXT_PUBLIC_TEXT_GENERATION_TIMEOUT_MS=${TEXT_GENERATION_TIMEOUT_MS}
+export NEXT_PUBLIC_CSP_WHITELIST=${CSP_WHITELIST}

 pm2 start ./pm2.json --no-daemon
--- a/web/middleware.ts
+++ b/web/middleware.ts
@ -0,0 +1,76 @@
+import type { NextRequest } from 'next/server'
+import { NextResponse } from 'next/server'
+
+const NECESSARY_DOMAIN = '*.sentry.io http://localhost:* http://127.0.0.1:* https://analytics.google.com https://googletagmanager.com https://api.github.com'
+
+export function middleware(request: NextRequest) {
+  const isWhiteListEnabled = !!process.env.NEXT_PUBLIC_CSP_WHITELIST && process.env.NODE_ENV === 'production'
+  if (!isWhiteListEnabled)
+    return NextResponse.next()
+
+  const whiteList = `${process.env.NEXT_PUBLIC_CSP_WHITELIST} ${NECESSARY_DOMAIN}`
+  const nonce = Buffer.from(crypto.randomUUID()).toString('base64')
+  const csp = `'nonce-${nonce}'`
+
+  const scheme_source = 'data: mediastream: blob: filesystem:'
+
+  const cspHeader = `
+    default-src 'self' ${scheme_source} ${csp} ${whiteList};
+    connect-src 'self' ${scheme_source} ${csp} ${whiteList};
+    script-src 'self' ${scheme_source} ${csp} ${whiteList};
+    style-src 'self' 'unsafe-inline' ${scheme_source} ${whiteList};
+    worker-src 'self' ${scheme_source} ${csp} ${whiteList};
+    media-src 'self' ${scheme_source} ${csp} ${whiteList};
+    img-src 'self' ${scheme_source} ${csp} ${whiteList};
+    font-src 'self';
+    object-src 'none';
+    base-uri 'self';
+    form-action 'self';
+    upgrade-insecure-requests;
+`
+  // Replace newline characters and spaces
+  const contentSecurityPolicyHeaderValue = cspHeader
+    .replace(/\s{2,}/g, ' ')
+    .trim()
+
+  const requestHeaders = new Headers(request.headers)
+  requestHeaders.set('x-nonce', nonce)
+
+  requestHeaders.set(
+    'Content-Security-Policy',
+    contentSecurityPolicyHeaderValue,
+  )
+
+  const response = NextResponse.next({
+    request: {
+      headers: requestHeaders,
+    },
+  })
+  response.headers.set(
+    'Content-Security-Policy',
+    contentSecurityPolicyHeaderValue,
+  )
+
+  return response
+}
+
+export const config = {
+  matcher: [
+    /*
+     * Match all request paths except for the ones starting with:
+     * - api (API routes)
+     * - _next/static (static files)
+     * - _next/image (image optimization files)
+     * - favicon.ico (favicon file)
+     */
+    {
+      // source: '/((?!api|_next/static|_next/image|favicon.ico).*)',
+      source: '/((?!_next/static|_next/image|favicon.ico).*)',
+      // source: '/(.*)',
+      // missing: [
+      //   { type: 'header', key: 'next-router-prefetch' },
+      //   { type: 'header', key: 'purpose', value: 'prefetch' },
+      // ],
+    },
+  ],
+}
--- a/web/package.json
+++ b/web/package.json
@ -63,7 +63,6 @@
    "mermaid": "10.4.0",
    "negotiator": "^0.6.3",
    "next": "^14.1.1",
-    "next-nprogress-bar": "^2.3.8",
    "pinyin-pro": "^3.23.0",
    "qrcode.react": "^3.1.0",
    "qs": "^6.11.1",
--- a/web/yarn.lock
+++ b/web/yarn.lock
@ -7342,18 +7342,6 @@ negotiator@^0.6.3:
  resolved "https://registry.npmjs.org/negotiator/-/negotiator-0.6.3.tgz"
  integrity sha512-+EUsqGPLsM+j/zdChZjsnX51g4XrHFOIXwfnCVPGlQk/k5giakcKsuxCObBRu6DSm9opw/O6slWbJdghQM4bBg==

-neo-async@^2.6.2:
-  version "2.6.2"
-  resolved "https://registry.npmjs.org/neo-async/-/neo-async-2.6.2.tgz"
-  integrity sha512-Yd3UES5mWCSqR+qNT93S3UoYUkqAZ9lLg8a7g9rimsWmYGK8cVToA4/sF3RrshdyV3sAGMXVUmpMYOw+dLpOuw==
-
-next-nprogress-bar@^2.3.8:
-  version "2.3.14"
-  resolved "https://registry.npmjs.org/next-nprogress-bar/-/next-nprogress-bar-2.3.14.tgz"
-  integrity sha512-r2zdo5SFakm1CSYLBo2/+9X2F5NRnbV/LI1b/Iu3mFLa9ln+Dlwx3vjJc3kVHI4i42jRC0an++obVHeWkYU6gA==
-  dependencies:
-    nprogress "^0.2.0"
-
 next@^14.1.1:
  version "14.2.4"
  resolved "https://registry.npmjs.org/next/-/next-14.2.4.tgz"
@ -7436,11 +7424,6 @@ npm-run-path@^5.1.0:
  dependencies:
    path-key "^4.0.0"

-nprogress@^0.2.0:
-  version "0.2.0"
-  resolved "https://registry.npmjs.org/nprogress/-/nprogress-0.2.0.tgz"
-  integrity sha512-I19aIingLgR1fmhftnbWWO3dXc0hSxqHQHQb3H8m+K3TnEn/iSeTZZOyvKXWqQESMwuUVnatlCnZdLBZZt2VSA==
-
 nth-check@^2.0.1:
  version "2.1.1"
  resolved "https://registry.npmjs.org/nth-check/-/nth-check-2.1.1.tgz"
@ -8974,6 +8957,14 @@ stringify-entities@^4.0.0:
  dependencies:
    ansi-regex "^5.0.1"

+strip-ansi@^6.0.0, strip-ansi@^6.0.1:
+"strip-ansi-cjs@npm:strip-ansi@^6.0.1":
+  version "6.0.1"
+  resolved "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz"
+  integrity sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==
+  dependencies:
+    ansi-regex "^5.0.1"
+
 strip-ansi@^6.0.0, strip-ansi@^6.0.1:
  version "6.0.1"
  resolved "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz"
@ -9866,6 +9857,7 @@ word-wrap@^1.2.3:
  resolved "https://registry.npmjs.org/word-wrap/-/word-wrap-1.2.5.tgz"
  integrity sha512-BN22B5eaMMI9UMtjrGd5g5eCYPpCPDUy0FJXbYsaT5zYxjFOckS53SQDE3pWkVoWpHXVb3BrYcEN4Twa55B5cA==

+"wrap-ansi-cjs@npm:wrap-ansi@^7.0.0":
 "wrap-ansi-cjs@npm:wrap-ansi@^7.0.0":
  version "7.0.0"
  resolved "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-7.0.0.tgz"
@ -9893,6 +9885,15 @@ wrap-ansi@^7.0.0:
    string-width "^4.1.0"
    strip-ansi "^6.0.0"

+wrap-ansi@^7.0.0:
+  version "7.0.0"
+  resolved "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-7.0.0.tgz"
+  integrity sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q==
+  dependencies:
+    ansi-styles "^4.0.0"
+    string-width "^4.1.0"
+    strip-ansi "^6.0.0"
+
 wrap-ansi@^8.1.0:
  version "8.1.0"
  resolved "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-8.1.0.tgz"