feat: support csp (#9111)

Co-authored-by: Joel <iamjoel007@gmail.com>

docker/.env.example

@@ -798,3 +798,5 @@ POSITION_TOOL_EXCLUDES=
 POSITION_PROVIDER_PINS=
 POSITION_PROVIDER_INCLUDES=
 POSITION_PROVIDER_EXCLUDES=
+# CSP https://developer.mozilla.org/en-US/docs/Web/HTTP/CSP
+CSP_WHITELIST=

docker/docker-compose.yaml

@@ -261,6 +261,7 @@ services:
       SENTRY_DSN: ${WEB_SENTRY_DSN:-}
       NEXT_TELEMETRY_DISABLED: ${NEXT_TELEMETRY_DISABLED:-0}
       TEXT_GENERATION_TIMEOUT_MS: ${TEXT_GENERATION_TIMEOUT_MS:-60000}
+      CSP_WHITELIST: ${CSP_WHITELIST:-}
 
   # The postgres database.
   db:
@@ -280,7 +281,7 @@ services:
     volumes:
       - ./volumes/db/data:/var/lib/postgresql/data
     healthcheck:
-      test: [ "CMD", "pg_isready" ]
+      test: ['CMD', 'pg_isready']
       interval: 1s
       timeout: 3s
       retries: 30
@@ -295,7 +296,7 @@ services:
     # Set the redis password when startup redis server.
     command: redis-server --requirepass ${REDIS_PASSWORD:-difyai123456}
     healthcheck:
-      test: [ "CMD", "redis-cli", "ping" ]
+      test: ['CMD', 'redis-cli', 'ping']
 
   # The DifySandbox
   sandbox:
@@ -315,7 +316,7 @@ services:
     volumes:
       - ./volumes/sandbox/dependencies:/dependencies
     healthcheck:
-      test: [ "CMD", "curl", "-f", "http://localhost:8194/health" ]
+      test: ['CMD', 'curl', '-f', 'http://localhost:8194/health']
     networks:
       - ssrf_proxy_network
 
@@ -328,7 +329,12 @@ services:
     volumes:
       - ./ssrf_proxy/squid.conf.template:/etc/squid/squid.conf.template
       - ./ssrf_proxy/docker-entrypoint.sh:/docker-entrypoint-mount.sh
-    entrypoint: [ "sh", "-c", "cp /docker-entrypoint-mount.sh /docker-entrypoint.sh && sed -i 's/\r$$//' /docker-entrypoint.sh && chmod +x /docker-entrypoint.sh && /docker-entrypoint.sh" ]
+    entrypoint:
+      [
+        'sh',
+        '-c',
+        "cp /docker-entrypoint-mount.sh /docker-entrypoint.sh && sed -i 's/\r$$//' /docker-entrypoint.sh && chmod +x /docker-entrypoint.sh && /docker-entrypoint.sh",
+      ]
     environment:
       # pls clearly modify the squid env vars to fit your network environment.
       HTTP_PORT: ${SSRF_HTTP_PORT:-3128}
@@ -357,8 +363,8 @@ services:
       - CERTBOT_EMAIL=${CERTBOT_EMAIL}
       - CERTBOT_DOMAIN=${CERTBOT_DOMAIN}
       - CERTBOT_OPTIONS=${CERTBOT_OPTIONS:-}
-    entrypoint: [ "/docker-entrypoint.sh" ]
+    entrypoint: ['/docker-entrypoint.sh']
-    command: [ "tail", "-f", "/dev/null" ]
+    command: ['tail', '-f', '/dev/null']
 
   # The nginx reverse proxy.
   # used for reverse proxying the API service and Web service.
@@ -375,7 +381,12 @@ services:
       - ./volumes/certbot/conf/live:/etc/letsencrypt/live # cert dir (with certbot container)
       - ./volumes/certbot/conf:/etc/letsencrypt
       - ./volumes/certbot/www:/var/www/html
-    entrypoint: [ "sh", "-c", "cp /docker-entrypoint-mount.sh /docker-entrypoint.sh && sed -i 's/\r$$//' /docker-entrypoint.sh && chmod +x /docker-entrypoint.sh && /docker-entrypoint.sh" ]
+    entrypoint:
+      [
+        'sh',
+        '-c',
+        "cp /docker-entrypoint-mount.sh /docker-entrypoint.sh && sed -i 's/\r$$//' /docker-entrypoint.sh && chmod +x /docker-entrypoint.sh && /docker-entrypoint.sh",
+      ]
     environment:
       NGINX_SERVER_NAME: ${NGINX_SERVER_NAME:-_}
       NGINX_HTTPS_ENABLED: ${NGINX_HTTPS_ENABLED:-false}
@@ -397,14 +408,14 @@ services:
       - api
       - web
     ports:
-      - "${EXPOSE_NGINX_PORT:-80}:${NGINX_PORT:-80}"
+      - '${EXPOSE_NGINX_PORT:-80}:${NGINX_PORT:-80}'
-      - "${EXPOSE_NGINX_SSL_PORT:-443}:${NGINX_SSL_PORT:-443}"
+      - '${EXPOSE_NGINX_SSL_PORT:-443}:${NGINX_SSL_PORT:-443}'
 
   # The Weaviate vector store.
   weaviate:
     image: semitechnologies/weaviate:1.19.0
     profiles:
-      - ""
+      - ''
       - weaviate
     restart: always
     volumes:
@@ -453,7 +464,7 @@ services:
     volumes:
       - ./volumes/pgvector/data:/var/lib/postgresql/data
     healthcheck:
-      test: [ "CMD", "pg_isready" ]
+      test: ['CMD', 'pg_isready']
       interval: 1s
       timeout: 3s
       retries: 30
@@ -475,7 +486,7 @@ services:
     volumes:
       - ./volumes/pgvecto_rs/data:/var/lib/postgresql/data
     healthcheck:
-      test: [ "CMD", "pg_isready" ]
+      test: ['CMD', 'pg_isready']
       interval: 1s
       timeout: 3s
       retries: 30
@@ -523,7 +534,7 @@ services:
       - ./volumes/milvus/etcd:/etcd
     command: etcd -advertise-client-urls=http://127.0.0.1:2379 -listen-client-urls http://0.0.0.0:2379 --data-dir /etcd
     healthcheck:
-      test: [ "CMD", "etcdctl", "endpoint", "health" ]
+      test: ['CMD', 'etcdctl', 'endpoint', 'health']
       interval: 30s
       timeout: 20s
       retries: 3
@@ -542,7 +553,7 @@ services:
       - ./volumes/milvus/minio:/minio_data
     command: minio server /minio_data --console-address ":9001"
     healthcheck:
-      test: [ "CMD", "curl", "-f", "http://localhost:9000/minio/health/live" ]
+      test: ['CMD', 'curl', '-f', 'http://localhost:9000/minio/health/live']
       interval: 30s
       timeout: 20s
       retries: 3
@@ -554,7 +565,7 @@ services:
     image: milvusdb/milvus:v2.3.1
     profiles:
       - milvus
-    command: [ "milvus", "run", "standalone" ]
+    command: ['milvus', 'run', 'standalone']
     environment:
       ETCD_ENDPOINTS: ${ETCD_ENDPOINTS:-etcd:2379}
       MINIO_ADDRESS: ${MINIO_ADDRESS:-minio:9000}
@@ -562,7 +573,7 @@ services:
     volumes:
       - ./volumes/milvus/milvus:/var/lib/milvus
     healthcheck:
-      test: [ "CMD", "curl", "-f", "http://localhost:9091/healthz" ]
+      test: ['CMD', 'curl', '-f', 'http://localhost:9091/healthz']
       interval: 30s
       start_period: 90s
       timeout: 20s
@@ -644,13 +655,13 @@ services:
       node.name: dify-es0
       discovery.type: single-node
       xpack.license.self_generated.type: trial
-      xpack.security.enabled: "true"
+      xpack.security.enabled: 'true'
-      xpack.security.enrollment.enabled: "false"
+      xpack.security.enrollment.enabled: 'false'
-      xpack.security.http.ssl.enabled: "false"
+      xpack.security.http.ssl.enabled: 'false'
     ports:
       - ${ELASTICSEARCH_PORT:-9200}:9200
     healthcheck:
-      test: [ "CMD", "curl", "-s", "http://localhost:9200/_cluster/health?pretty" ]
+      test: ['CMD', 'curl', '-s', 'http://localhost:9200/_cluster/health?pretty']
       interval: 30s
       timeout: 10s
       retries: 50
@@ -668,17 +679,17 @@ services:
     environment:
       XPACK_ENCRYPTEDSAVEDOBJECTS_ENCRYPTIONKEY: d1a66dfd-c4d3-4a0a-8290-2abcb83ab3aa
       NO_PROXY: localhost,127.0.0.1,elasticsearch,kibana
-      XPACK_SECURITY_ENABLED: "true"
+      XPACK_SECURITY_ENABLED: 'true'
-      XPACK_SECURITY_ENROLLMENT_ENABLED: "false"
+      XPACK_SECURITY_ENROLLMENT_ENABLED: 'false'
-      XPACK_SECURITY_HTTP_SSL_ENABLED: "false"
+      XPACK_SECURITY_HTTP_SSL_ENABLED: 'false'
-      XPACK_FLEET_ISAIRGAPPED: "true"
+      XPACK_FLEET_ISAIRGAPPED: 'true'
       I18N_LOCALE: zh-CN
-      SERVER_PORT: "5601"
+      SERVER_PORT: '5601'
       ELASTICSEARCH_HOSTS: http://elasticsearch:9200
     ports:
       - ${KIBANA_PORT:-5601}:5601
     healthcheck:
-      test: [ "CMD-SHELL", "curl -s http://localhost:5601 >/dev/null || exit 1" ]
+      test: ['CMD-SHELL', 'curl -s http://localhost:5601 >/dev/null || exit 1']
       interval: 30s
       timeout: 10s
       retries: 3

web/.env.example

@@ -22,3 +22,6 @@ NEXT_PUBLIC_UPLOAD_IMAGE_AS_ICON=false
 
 # The timeout for the text generation in millisecond
 NEXT_PUBLIC_TEXT_GENERATION_TIMEOUT_MS=60000
+
+# CSP https://developer.mozilla.org/en-US/docs/Web/HTTP/CSP
+NEXT_PUBLIC_CSP_WHITELIST=

web/app/components/base/ga/index.tsx

@@ -1,6 +1,7 @@
 import type { FC } from 'react'
 import React from 'react'
 import Script from 'next/script'
+import { headers } from 'next/headers'
 import { IS_CE_EDITION } from '@/config'
 
 export enum GaType {
@@ -23,9 +24,16 @@ const GA: FC<IGAProps> = ({
   if (IS_CE_EDITION)
     return null
 
+  const nonce = process.env.NODE_ENV === 'production' ? headers().get('x-nonce') : ''
+
   return (
     <>
-      <Script strategy="beforeInteractive" async src={`https://www.googletagmanager.com/gtag/js?id=${gaIdMaps[gaType]}`}></Script>
+      <Script
+        strategy="beforeInteractive"
+        async
+        src={`https://www.googletagmanager.com/gtag/js?id=${gaIdMaps[gaType]}`}
+        nonce={nonce!}
+      ></Script>
       <Script
         id="ga-init"
         dangerouslySetInnerHTML={{
@@ -36,6 +44,7 @@ gtag('js', new Date());
 gtag('config', '${gaIdMaps[gaType]}');
           `,
         }}
+        nonce={nonce!}
       >
       </Script>
     </>

web/app/components/base/topbar/index.tsx

@@ -1,16 +0,0 @@
-'use client'
-
-import { AppProgressBar as ProgressBar } from 'next-nprogress-bar'
-
-const Topbar = () => {
-  return (
-    <>
-      <ProgressBar
-        height='2px'
-        color="#1C64F2FF"
-        options={{ showSpinner: false }}
-        shallowRouting />
-    </>)
-}
-
-export default Topbar

web/app/layout.tsx

@@ -2,7 +2,6 @@ import type { Viewport } from 'next'
 import I18nServer from './components/i18n-server'
 import BrowserInitor from './components/browser-initor'
 import SentryInitor from './components/sentry-initor'
-import Topbar from './components/base/topbar'
 import { getLocaleOnServer } from '@/i18n/server'
 import './styles/globals.css'
 import './styles/markdown.scss'
@@ -45,7 +44,6 @@ const LocaleLayout = ({
         data-public-site-about={process.env.NEXT_PUBLIC_SITE_ABOUT}
         data-public-text-generation-timeout-ms={process.env.NEXT_PUBLIC_TEXT_GENERATION_TIMEOUT_MS}
       >
-        <Topbar />
         <BrowserInitor>
           <SentryInitor>
             <I18nServer>{children}</I18nServer>

web/docker/entrypoint.sh

@@ -22,5 +22,6 @@ export NEXT_PUBLIC_SITE_ABOUT=${SITE_ABOUT}
 export NEXT_TELEMETRY_DISABLED=${NEXT_TELEMETRY_DISABLED}
 
 export NEXT_PUBLIC_TEXT_GENERATION_TIMEOUT_MS=${TEXT_GENERATION_TIMEOUT_MS}
+export NEXT_PUBLIC_CSP_WHITELIST=${CSP_WHITELIST}
 
 pm2 start ./pm2.json --no-daemon

web/middleware.ts

@@ -0,0 +1,76 @@
+import type { NextRequest } from 'next/server'
+import { NextResponse } from 'next/server'
+
+const NECESSARY_DOMAIN = '*.sentry.io http://localhost:* http://127.0.0.1:* https://analytics.google.com https://googletagmanager.com https://api.github.com'
+
+export function middleware(request: NextRequest) {
+  const isWhiteListEnabled = !!process.env.NEXT_PUBLIC_CSP_WHITELIST && process.env.NODE_ENV === 'production'
+  if (!isWhiteListEnabled)
+    return NextResponse.next()
+
+  const whiteList = `${process.env.NEXT_PUBLIC_CSP_WHITELIST} ${NECESSARY_DOMAIN}`
+  const nonce = Buffer.from(crypto.randomUUID()).toString('base64')
+  const csp = `'nonce-${nonce}'`
+
+  const scheme_source = 'data: mediastream: blob: filesystem:'
+
+  const cspHeader = `
+    default-src 'self' ${scheme_source} ${csp} ${whiteList};
+    connect-src 'self' ${scheme_source} ${csp} ${whiteList};
+    script-src 'self' ${scheme_source} ${csp} ${whiteList};
+    style-src 'self' 'unsafe-inline' ${scheme_source} ${whiteList};
+    worker-src 'self' ${scheme_source} ${csp} ${whiteList};
+    media-src 'self' ${scheme_source} ${csp} ${whiteList};
+    img-src 'self' ${scheme_source} ${csp} ${whiteList};
+    font-src 'self';
+    object-src 'none';
+    base-uri 'self';
+    form-action 'self';
+    upgrade-insecure-requests;
+`
+  // Replace newline characters and spaces
+  const contentSecurityPolicyHeaderValue = cspHeader
+    .replace(/\s{2,}/g, ' ')
+    .trim()
+
+  const requestHeaders = new Headers(request.headers)
+  requestHeaders.set('x-nonce', nonce)
+
+  requestHeaders.set(
+    'Content-Security-Policy',
+    contentSecurityPolicyHeaderValue,
+  )
+
+  const response = NextResponse.next({
+    request: {
+      headers: requestHeaders,
+    },
+  })
+  response.headers.set(
+    'Content-Security-Policy',
+    contentSecurityPolicyHeaderValue,
+  )
+
+  return response
+}
+
+export const config = {
+  matcher: [
+    /*
+     * Match all request paths except for the ones starting with:
+     * - api (API routes)
+     * - _next/static (static files)
+     * - _next/image (image optimization files)
+     * - favicon.ico (favicon file)
+     */
+    {
+      // source: '/((?!api|_next/static|_next/image|favicon.ico).*)',
+      source: '/((?!_next/static|_next/image|favicon.ico).*)',
+      // source: '/(.*)',
+      // missing: [
+      //   { type: 'header', key: 'next-router-prefetch' },
+      //   { type: 'header', key: 'purpose', value: 'prefetch' },
+      // ],
+    },
+  ],
+}

web/package.json

@@ -63,7 +63,6 @@
     "mermaid": "10.4.0",
     "negotiator": "^0.6.3",
     "next": "^14.1.1",
-    "next-nprogress-bar": "^2.3.8",
     "pinyin-pro": "^3.23.0",
     "qrcode.react": "^3.1.0",
     "qs": "^6.11.1",

web/yarn.lock

@@ -7342,18 +7342,6 @@ negotiator@^0.6.3:
   resolved "https://registry.npmjs.org/negotiator/-/negotiator-0.6.3.tgz"
   integrity sha512-+EUsqGPLsM+j/zdChZjsnX51g4XrHFOIXwfnCVPGlQk/k5giakcKsuxCObBRu6DSm9opw/O6slWbJdghQM4bBg==
 
-neo-async@^2.6.2:
-  version "2.6.2"
-  resolved "https://registry.npmjs.org/neo-async/-/neo-async-2.6.2.tgz"
-  integrity sha512-Yd3UES5mWCSqR+qNT93S3UoYUkqAZ9lLg8a7g9rimsWmYGK8cVToA4/sF3RrshdyV3sAGMXVUmpMYOw+dLpOuw==
-
-next-nprogress-bar@^2.3.8:
-  version "2.3.14"
-  resolved "https://registry.npmjs.org/next-nprogress-bar/-/next-nprogress-bar-2.3.14.tgz"
-  integrity sha512-r2zdo5SFakm1CSYLBo2/+9X2F5NRnbV/LI1b/Iu3mFLa9ln+Dlwx3vjJc3kVHI4i42jRC0an++obVHeWkYU6gA==
-  dependencies:
-    nprogress "^0.2.0"
-
 next@^14.1.1:
   version "14.2.4"
   resolved "https://registry.npmjs.org/next/-/next-14.2.4.tgz"
@@ -7436,11 +7424,6 @@ npm-run-path@^5.1.0:
   dependencies:
     path-key "^4.0.0"
 
-nprogress@^0.2.0:
-  version "0.2.0"
-  resolved "https://registry.npmjs.org/nprogress/-/nprogress-0.2.0.tgz"
-  integrity sha512-I19aIingLgR1fmhftnbWWO3dXc0hSxqHQHQb3H8m+K3TnEn/iSeTZZOyvKXWqQESMwuUVnatlCnZdLBZZt2VSA==
-
 nth-check@^2.0.1:
   version "2.1.1"
   resolved "https://registry.npmjs.org/nth-check/-/nth-check-2.1.1.tgz"
@@ -8974,6 +8957,14 @@ stringify-entities@^4.0.0:
   dependencies:
     ansi-regex "^5.0.1"
 
+strip-ansi@^6.0.0, strip-ansi@^6.0.1:
+"strip-ansi-cjs@npm:strip-ansi@^6.0.1":
+  version "6.0.1"
+  resolved "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz"
+  integrity sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==
+  dependencies:
+    ansi-regex "^5.0.1"
+
 strip-ansi@^6.0.0, strip-ansi@^6.0.1:
   version "6.0.1"
   resolved "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz"
@@ -9866,6 +9857,7 @@ word-wrap@^1.2.3:
   resolved "https://registry.npmjs.org/word-wrap/-/word-wrap-1.2.5.tgz"
   integrity sha512-BN22B5eaMMI9UMtjrGd5g5eCYPpCPDUy0FJXbYsaT5zYxjFOckS53SQDE3pWkVoWpHXVb3BrYcEN4Twa55B5cA==
 
+"wrap-ansi-cjs@npm:wrap-ansi@^7.0.0":
 "wrap-ansi-cjs@npm:wrap-ansi@^7.0.0":
   version "7.0.0"
   resolved "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-7.0.0.tgz"
@@ -9893,6 +9885,15 @@ wrap-ansi@^7.0.0:
     string-width "^4.1.0"
     strip-ansi "^6.0.0"
 
+wrap-ansi@^7.0.0:
+  version "7.0.0"
+  resolved "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-7.0.0.tgz"
+  integrity sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q==
+  dependencies:
+    ansi-styles "^4.0.0"
+    string-width "^4.1.0"
+    strip-ansi "^6.0.0"
+
 wrap-ansi@^8.1.0:
   version "8.1.0"
   resolved "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-8.1.0.tgz"