Chore/remove python dependencies selector (#7494)

api/core/helper/code_executor/code_executor.py

@@ -1,15 +1,13 @@
 import logging
-import time
 from enum import Enum
 from threading import Lock
-from typing import Literal, Optional
+from typing import Optional
 
-from httpx import Timeout, get, post
+from httpx import Timeout, post
 from pydantic import BaseModel
 from yarl import URL
 
 from configs import dify_config
-from core.helper.code_executor.entities import CodeDependency
 from core.helper.code_executor.javascript.javascript_transformer import NodeJsTemplateTransformer
 from core.helper.code_executor.jinja2.jinja2_transformer import Jinja2TemplateTransformer
 from core.helper.code_executor.python3.python3_transformer import Python3TemplateTransformer
@@ -66,8 +64,7 @@ class CodeExecutor:
     def execute_code(cls, 
                      language: CodeLanguage, 
                      preload: str, 
-                     code: str, 
+                     code: str) -> str:
-                     dependencies: Optional[list[CodeDependency]] = None) -> str:
         """
         Execute code
         :param language: code language
@@ -87,9 +84,6 @@ class CodeExecutor:
             'enable_network': True
         }
 
-        if dependencies:
-            data['dependencies'] = [dependency.model_dump() for dependency in dependencies]
-
         try:
             response = post(str(url), json=data, headers=headers, timeout=CODE_EXECUTION_TIMEOUT)
             if response.status_code == 503:
@@ -119,7 +113,7 @@ class CodeExecutor:
         return response.data.stdout or ''
 
     @classmethod
-    def execute_workflow_code_template(cls, language: CodeLanguage, code: str, inputs: dict, dependencies: Optional[list[CodeDependency]] = None) -> dict:
+    def execute_workflow_code_template(cls, language: CodeLanguage, code: str, inputs: dict) -> dict:
         """
         Execute code
         :param language: code language
@@ -131,67 +125,12 @@ class CodeExecutor:
         if not template_transformer:
             raise CodeExecutionException(f'Unsupported language {language}')
 
-        runner, preload, dependencies = template_transformer.transform_caller(code, inputs, dependencies)
+        runner, preload = template_transformer.transform_caller(code, inputs)
 
         try:
-            response = cls.execute_code(language, preload, runner, dependencies)
+            response = cls.execute_code(language, preload, runner)
         except CodeExecutionException as e:
             raise e
 
         return template_transformer.transform_response(response)
-    
+    
-    @classmethod
-    def list_dependencies(cls, language: str) -> list[CodeDependency]:
-        if language not in cls.supported_dependencies_languages:
-            return []
-
-        with cls.dependencies_cache_lock:
-            if language in cls.dependencies_cache:
-                # check expiration
-                dependencies = cls.dependencies_cache[language]
-                if dependencies['expiration'] > time.time():
-                    return dependencies['data']
-                # remove expired cache
-                del cls.dependencies_cache[language]
-        
-        dependencies = cls._get_dependencies(language)
-        with cls.dependencies_cache_lock:
-            cls.dependencies_cache[language] = {
-                'data': dependencies,
-                'expiration': time.time() + 60
-            }
-        
-        return dependencies
-        
-    @classmethod
-    def _get_dependencies(cls, language: Literal['python3']) -> list[CodeDependency]:
-        """
-        List dependencies
-        """
-        url = URL(CODE_EXECUTION_ENDPOINT) / 'v1' / 'sandbox' / 'dependencies'
-
-        headers = {
-            'X-Api-Key': CODE_EXECUTION_API_KEY
-        }
-
-        running_language = cls.code_language_to_running_language.get(language)
-        if isinstance(running_language, Enum):
-            running_language = running_language.value
-
-        data = {
-            'language': running_language,
-        }
-
-        try:
-            response = get(str(url), params=data, headers=headers, timeout=CODE_EXECUTION_TIMEOUT)
-            if response.status_code != 200:
-                raise Exception(f'Failed to list dependencies, got status code {response.status_code}, please check if the sandbox service is running')
-            response = response.json()
-            dependencies = response.get('data', {}).get('dependencies', [])
-            return [
-                CodeDependency(**dependency) for dependency in dependencies
-                if dependency.get('name') not in Python3TemplateTransformer.get_standard_packages()
-            ]
-        except Exception as e:
-            logger.exception(f'Failed to list dependencies: {e}')
-            return []

api/core/helper/code_executor/code_node_provider.py

@@ -2,8 +2,6 @@ from abc import abstractmethod
 
 from pydantic import BaseModel
 
-from core.helper.code_executor.code_executor import CodeExecutor
-
 
 class CodeNodeProvider(BaseModel):
     @staticmethod
@@ -23,10 +21,6 @@ class CodeNodeProvider(BaseModel):
         """
         pass
 
-    @classmethod
-    def get_default_available_packages(cls) -> list[dict]:
-        return [p.model_dump() for p in CodeExecutor.list_dependencies(cls.get_language())]
-
     @classmethod
     def get_default_config(cls) -> dict:
         return {
@@ -50,6 +44,5 @@ class CodeNodeProvider(BaseModel):
                         "children": None
                     }
                 }
-            },
+            }
-            "available_dependencies": cls.get_default_available_packages(),
         }

api/core/helper/code_executor/entities.py

@@ -1,6 +0,0 @@
-from pydantic import BaseModel
-
-
-class CodeDependency(BaseModel):
-    name: str
-    version: str

api/core/helper/code_executor/jinja2/jinja2_formatter.py

@@ -3,7 +3,7 @@ from core.helper.code_executor.code_executor import CodeExecutor, CodeLanguage
 
 class Jinja2Formatter:
     @classmethod
-    def format(cls, template: str, inputs: str) -> str:
+    def format(cls, template: str, inputs: dict) -> str:
         """
         Format template
         :param template: template

api/core/helper/code_executor/jinja2/jinja2_transformer.py

@@ -1,14 +1,9 @@
 from textwrap import dedent
 
-from core.helper.code_executor.python3.python3_transformer import Python3TemplateTransformer
 from core.helper.code_executor.template_transformer import TemplateTransformer
 
 
 class Jinja2TemplateTransformer(TemplateTransformer):
-    @classmethod
-    def get_standard_packages(cls) -> set[str]:
-        return {'jinja2'} | Python3TemplateTransformer.get_standard_packages()
-
     @classmethod
     def transform_response(cls, response: str) -> dict:
         """

api/core/helper/code_executor/python3/python3_transformer.py

@@ -4,30 +4,6 @@ from core.helper.code_executor.template_transformer import TemplateTransformer
 
 
 class Python3TemplateTransformer(TemplateTransformer):
-    @classmethod
-    def get_standard_packages(cls) -> set[str]:
-        return {
-            'base64',
-            'binascii',
-            'collections',
-            'datetime',
-            'functools',
-            'hashlib',
-            'hmac',
-            'itertools',
-            'json',
-            'math',
-            'operator',
-            'os',
-            'random',
-            're',
-            'string',
-            'sys',
-            'time',
-            'traceback',
-            'uuid',
-        }
-
     @classmethod
     def get_runner_script(cls) -> str:
         runner_script = dedent(f"""

api/core/helper/code_executor/template_transformer.py

@@ -2,9 +2,6 @@ import json
 import re
 from abc import ABC, abstractmethod
 from base64 import b64encode
-from typing import Optional
-
-from core.helper.code_executor.entities import CodeDependency
 
 
 class TemplateTransformer(ABC):
@@ -13,12 +10,7 @@ class TemplateTransformer(ABC):
     _result_tag: str = '<<RESULT>>'
 
     @classmethod
-    def get_standard_packages(cls) -> set[str]:
+    def transform_caller(cls, code: str, inputs: dict) -> tuple[str, str]:
-        return set()
-
-    @classmethod
-    def transform_caller(cls, code: str, inputs: dict,
-                         dependencies: Optional[list[CodeDependency]] = None) -> tuple[str, str, list[CodeDependency]]:
         """
         Transform code to python runner
         :param code: code
@@ -28,14 +20,7 @@ class TemplateTransformer(ABC):
         runner_script = cls.assemble_runner_script(code, inputs)
         preload_script = cls.get_preload_script()
 
-        packages = dependencies or []
+        return runner_script, preload_script
-        standard_packages = cls.get_standard_packages()
-        for package in standard_packages:
-            if package not in packages:
-                packages.append(CodeDependency(name=package, version=''))
-        packages = list({dep.name: dep for dep in packages if dep.name}.values())
-
-        return runner_script, preload_script, packages
 
     @classmethod
     def extract_result_str_from_response(cls, response: str) -> str:

api/core/workflow/nodes/code/code_node.py

@@ -67,7 +67,6 @@ class CodeNode(BaseNode):
                 language=code_language,
                 code=code,
                 inputs=variables,
-                dependencies=node_data.dependencies
             )
 
             # Transform result

api/core/workflow/nodes/code/entities.py

@@ -3,7 +3,6 @@ from typing import Literal, Optional
 from pydantic import BaseModel
 
 from core.helper.code_executor.code_executor import CodeLanguage
-from core.helper.code_executor.entities import CodeDependency
 from core.workflow.entities.base_node_data_entities import BaseNodeData
 from core.workflow.entities.variable_entities import VariableSelector
 
@@ -16,8 +15,12 @@ class CodeNodeData(BaseNodeData):
         type: Literal['string', 'number', 'object', 'array[string]', 'array[number]', 'array[object]']
         children: Optional[dict[str, 'Output']] = None
 
+    class Dependency(BaseModel):
+        name: str
+        version: str
+
     variables: list[VariableSelector]
     code_language: Literal[CodeLanguage.PYTHON3, CodeLanguage.JAVASCRIPT]
     code: str
     outputs: dict[str, Output]
-    dependencies: Optional[list[CodeDependency]] = None
+    dependencies: Optional[list[Dependency]] = None

api/tests/integration_tests/workflow/nodes/__mock/code_executor.py

@@ -6,14 +6,13 @@ from _pytest.monkeypatch import MonkeyPatch
 from jinja2 import Template
 
 from core.helper.code_executor.code_executor import CodeExecutor, CodeLanguage
-from core.helper.code_executor.entities import CodeDependency
 
 MOCK = os.getenv('MOCK_SWITCH', 'false') == 'true'
 
 class MockedCodeExecutor:
     @classmethod
     def invoke(cls, language: Literal['python3', 'javascript', 'jinja2'], 
-               code: str, inputs: dict, dependencies: Optional[list[CodeDependency]] = None) -> dict:
+               code: str, inputs: dict) -> dict:
         # invoke directly
         match language:
             case CodeLanguage.PYTHON3:
@@ -24,6 +23,8 @@ class MockedCodeExecutor:
                 return {
                     "result": Template(code).render(inputs)
                 }
+            case _:
+                raise Exception("Language not supported")
 
 @pytest.fixture
 def setup_code_executor_mock(request, monkeypatch: MonkeyPatch):

api/tests/integration_tests/workflow/nodes/code_executor/test_code_javascript.py

@@ -28,14 +28,6 @@ def test_javascript_with_code_template():
         inputs={'arg1': 'Hello', 'arg2': 'World'})
     assert result == {'result': 'HelloWorld'}
 
-
-def test_javascript_list_default_available_packages():
-    packages = JavascriptCodeProvider.get_default_available_packages()
-
-    # no default packages available for javascript
-    assert len(packages) == 0
-
-
 def test_javascript_get_runner_script():
     runner_script = NodeJsTemplateTransformer.get_runner_script()
     assert runner_script.count(NodeJsTemplateTransformer._code_placeholder) == 1

api/tests/integration_tests/workflow/nodes/code_executor/test_code_python3.py

@@ -29,15 +29,6 @@ def test_python3_with_code_template():
     assert result == {'result': 'HelloWorld'}
 
 
-def test_python3_list_default_available_packages():
-    packages = Python3CodeProvider.get_default_available_packages()
-    assert len(packages) > 0
-    assert {'requests', 'httpx'}.issubset(p['name'] for p in packages)
-
-    # check JSON serializable
-    assert len(str(json.dumps(packages))) > 0
-
-
 def test_python3_get_runner_script():
     runner_script = Python3TemplateTransformer.get_runner_script()
     assert runner_script.count(Python3TemplateTransformer._code_placeholder) == 1

web/app/components/workflow/nodes/code/dependency-picker.tsx

@@ -1,97 +0,0 @@
-import type { FC } from 'react'
-import React, { useCallback, useState } from 'react'
-import { t } from 'i18next'
-import {
-  RiArrowDownSLine,
-  RiSearchLine,
-} from '@remixicon/react'
-import type { CodeDependency } from './types'
-import { PortalToFollowElem, PortalToFollowElemContent, PortalToFollowElemTrigger } from '@/app/components/base/portal-to-follow-elem'
-import { Check } from '@/app/components/base/icons/src/vender/line/general'
-import { XCircle } from '@/app/components/base/icons/src/vender/solid/general'
-
-type Props = {
-  value: CodeDependency
-  available_dependencies: CodeDependency[]
-  onChange: (dependency: CodeDependency) => void
-}
-
-const DependencyPicker: FC<Props> = ({
-  available_dependencies,
-  value,
-  onChange,
-}) => {
-  const [open, setOpen] = useState(false)
-  const [searchText, setSearchText] = useState('')
-
-  const handleChange = useCallback((dependency: CodeDependency) => {
-    return () => {
-      setOpen(false)
-      onChange(dependency)
-    }
-  }, [onChange])
-
-  return (
-    <PortalToFollowElem
-      open={open}
-      onOpenChange={setOpen}
-      placement='bottom-start'
-      offset={4}
-    >
-      <PortalToFollowElemTrigger onClick={() => setOpen(!open)} className='flex-grow cursor-pointer'>
-        <div className='flex items-center h-8 justify-between px-2.5 rounded-lg border-0 bg-gray-100 text-gray-900 text-[13px]'>
-          <div className='grow w-0 truncate' title={value.name}>{value.name}</div>
-          <RiArrowDownSLine className='shrink-0 w-3.5 h-3.5 text-gray-700' />
-        </div>
-      </PortalToFollowElemTrigger>
-      <PortalToFollowElemContent style={{
-        zIndex: 100,
-      }}>
-        <div className='p-1 bg-white rounded-lg shadow-sm' style={{
-          width: 350,
-        }}>
-          <div
-            className='shadow-sm bg-white mb-2 mx-1 flex items-center px-2 rounded-lg bg-gray-100'
-          >
-            <RiSearchLine className='shrink-0 ml-[1px] mr-[5px] w-3.5 h-3.5 text-gray-400' />
-            <input
-              value={searchText}
-              className='grow px-0.5 py-[7px] text-[13px] text-gray-700 bg-transparent appearance-none outline-none caret-primary-600 placeholder:text-gray-400'
-              placeholder={t('workflow.nodes.code.searchDependencies') || ''}
-              onChange={e => setSearchText(e.target.value)}
-              autoFocus
-            />
-            {
-              searchText && (
-                <div
-                  className='flex items-center justify-center ml-[5px] w-[18px] h-[18px] cursor-pointer'
-                  onClick={() => setSearchText('')}
-                >
-                  <XCircle className='w-[14px] h-[14px] text-gray-400' />
-                </div>
-              )
-            }
-          </div>
-          <div className='max-h-[30vh] overflow-y-auto'>
-            {available_dependencies.filter((v) => {
-              if (!searchText)
-                return true
-              return v.name.toLowerCase().includes(searchText.toLowerCase())
-            }).map(dependency => (
-              <div
-                key={dependency.name}
-                className='flex items-center h-[30px] justify-between pl-3 pr-2 rounded-lg hover:bg-gray-100 text-gray-900 text-[13px] cursor-pointer'
-                onClick={handleChange(dependency)}
-              >
-                <div className='w-0 grow truncate'>{dependency.name}</div>
-                {dependency.name === value.name && <Check className='shrink-0 w-4 h-4 text-primary-600' />}
-              </div>
-            ))}
-          </div>
-        </div>
-      </PortalToFollowElemContent>
-    </PortalToFollowElem>
-  )
-}
-
-export default React.memo(DependencyPicker)

web/app/components/workflow/nodes/code/dependency.tsx

@@ -1,36 +0,0 @@
-import type { FC } from 'react'
-import React from 'react'
-import RemoveButton from '../_base/components/remove-button'
-import type { CodeDependency } from './types'
-import DependencyPicker from './dependency-picker'
-
-type Props = {
-  available_dependencies: CodeDependency[]
-  dependencies: CodeDependency[]
-  handleRemove: (index: number) => void
-  handleChange: (index: number, dependency: CodeDependency) => void
-}
-
-const Dependencies: FC<Props> = ({
-  available_dependencies, dependencies, handleRemove, handleChange,
-}) => {
-  return (
-    <div className='space-y-2'>
-      {dependencies.map((dependency, index) => (
-        <div className='flex items-center space-x-1' key={index}>
-          <DependencyPicker
-            value={dependency}
-            available_dependencies={available_dependencies}
-            onChange={dependency => handleChange(index, dependency)}
-          />
-          <RemoveButton
-            className='!p-2 !bg-gray-100 hover:!bg-gray-200'
-            onClick={() => handleRemove(index)}
-          />
-        </div>
-      ))}
-    </div>
-  )
-}
-
-export default React.memo(Dependencies)

web/app/components/workflow/nodes/code/panel.tsx

@@ -5,7 +5,6 @@ import RemoveEffectVarConfirm from '../_base/components/remove-effect-var-confir
 import useConfig from './use-config'
 import type { CodeNodeType } from './types'
 import { CodeLanguage } from './types'
-import Dependencies from './dependency'
 import VarList from '@/app/components/workflow/nodes/_base/components/variable/var-list'
 import OutputVarList from '@/app/components/workflow/nodes/_base/components/variable/output-var-list'
 import AddButton from '@/app/components/base/button/add-button'
@@ -60,11 +59,6 @@ const Panel: FC<NodePanelProps<CodeNodeType>> = ({
     varInputs,
     inputVarValues,
     setInputVarValues,
-    allowDependencies,
-    availableDependencies,
-    handleAddDependency,
-    handleRemoveDependency,
-    handleChangeDependency,
   } = useConfig(id, data)
 
   return (
@@ -84,31 +78,6 @@ const Panel: FC<NodePanelProps<CodeNodeType>> = ({
             filterVar={filterVar}
           />
         </Field>
-        {
-          allowDependencies
-            ? (
-              <div>
-                <Split />
-                <div className='pt-4'>
-                  <Field
-                    title={t(`${i18nPrefix}.advancedDependencies`)}
-                    operations={
-                      <AddButton onClick={() => handleAddDependency({ name: '', version: '' })} />
-                    }
-                    tooltip={t(`${i18nPrefix}.advancedDependenciesTip`)!}
-                  >
-                    <Dependencies
-                      available_dependencies={availableDependencies}
-                      dependencies={inputs.dependencies || []}
-                      handleRemove={index => handleRemoveDependency(index)}
-                      handleChange={(index, dependency) => handleChangeDependency(index, dependency)}
-                    />
-                  </Field>
-                </div>
-              </div>
-            )
-            : null
-        }
         <Split />
         <CodeEditor
           isInNode

web/app/components/workflow/nodes/code/types.ts

@@ -16,10 +16,4 @@ export type CodeNodeType = CommonNodeType & {
   code_language: CodeLanguage
   code: string
   outputs: OutputVar
-  dependencies?: CodeDependency[]
-}
-
-export type CodeDependency = {
-  name: string
-  version: string
 }

web/app/components/workflow/nodes/code/use-config.ts

@@ -5,7 +5,7 @@ import useOutputVarList from '../_base/hooks/use-output-var-list'
 import { BlockEnum, VarType } from '../../types'
 import type { Var } from '../../types'
 import { useStore } from '../../store'
-import type { CodeDependency, CodeNodeType, OutputVar } from './types'
+import type { CodeNodeType, OutputVar } from './types'
 import { CodeLanguage } from './types'
 import useNodeCrud from '@/app/components/workflow/nodes/_base/hooks/use-node-crud'
 import useOneStepRun from '@/app/components/workflow/nodes/_base/hooks/use-one-step-run'
@@ -21,19 +21,15 @@ const useConfig = (id: string, payload: CodeNodeType) => {
   const appId = useAppStore.getState().appDetail?.id
 
   const [allLanguageDefault, setAllLanguageDefault] = useState<Record<CodeLanguage, CodeNodeType> | null>(null)
-  const [allLanguageDependencies, setAllLanguageDependencies] = useState<Record<CodeLanguage, CodeDependency[]> | null>(null)
   useEffect(() => {
     if (appId) {
       (async () => {
         const { config: javaScriptConfig } = await fetchNodeDefault(appId, BlockEnum.Code, { code_language: CodeLanguage.javascript }) as any
-        const { config: pythonConfig, available_dependencies: pythonDependencies } = await fetchNodeDefault(appId, BlockEnum.Code, { code_language: CodeLanguage.python3 }) as any
+        const { config: pythonConfig } = await fetchNodeDefault(appId, BlockEnum.Code, { code_language: CodeLanguage.python3 }) as any
         setAllLanguageDefault({
           [CodeLanguage.javascript]: javaScriptConfig as CodeNodeType,
           [CodeLanguage.python3]: pythonConfig as CodeNodeType,
         } as any)
-        setAllLanguageDependencies({
-          [CodeLanguage.python3]: pythonDependencies as CodeDependency[],
-        } as any)
       })()
     }
   }, [appId])
@@ -45,62 +41,6 @@ const useConfig = (id: string, payload: CodeNodeType) => {
     setInputs,
   })
 
-  const handleAddDependency = useCallback((dependency: CodeDependency) => {
-    const newInputs = produce(inputs, (draft) => {
-      if (!draft.dependencies)
-        draft.dependencies = []
-      draft.dependencies.push(dependency)
-    })
-    setInputs(newInputs)
-  }, [inputs, setInputs])
-
-  const handleRemoveDependency = useCallback((index: number) => {
-    const newInputs = produce(inputs, (draft) => {
-      if (!draft.dependencies)
-        draft.dependencies = []
-      draft.dependencies.splice(index, 1)
-    })
-    setInputs(newInputs)
-  }, [inputs, setInputs])
-
-  const handleChangeDependency = useCallback((index: number, dependency: CodeDependency) => {
-    const newInputs = produce(inputs, (draft) => {
-      if (!draft.dependencies)
-        draft.dependencies = []
-      draft.dependencies[index] = dependency
-    })
-    setInputs(newInputs)
-  }, [inputs, setInputs])
-
-  const [allowDependencies, setAllowDependencies] = useState<boolean>(false)
-  useEffect(() => {
-    if (!inputs.code_language)
-      return
-    if (!allLanguageDependencies)
-      return
-
-    const newAllowDependencies = !!allLanguageDependencies[inputs.code_language]
-    setAllowDependencies(newAllowDependencies)
-  }, [allLanguageDependencies, inputs.code_language])
-
-  const [availableDependencies, setAvailableDependencies] = useState<CodeDependency[]>([])
-  useEffect(() => {
-    if (!inputs.code_language)
-      return
-    if (!allLanguageDependencies)
-      return
-
-    const newAvailableDependencies = produce(allLanguageDependencies[inputs.code_language], (draft) => {
-      const currentLanguage = inputs.code_language
-      if (!currentLanguage || !draft || !inputs.dependencies)
-        return []
-      return draft.filter((dependency) => {
-        return !inputs.dependencies?.find(d => d.name === dependency.name)
-      })
-    })
-    setAvailableDependencies(newAvailableDependencies || [])
-  }, [allLanguageDependencies, inputs.code_language, inputs.dependencies])
-
   const [outputKeyOrders, setOutputKeyOrders] = useState<string[]>([])
   const syncOutputKeyOrders = useCallback((outputs: OutputVar) => {
     setOutputKeyOrders(Object.keys(outputs))
@@ -223,11 +163,6 @@ const useConfig = (id: string, payload: CodeNodeType) => {
     inputVarValues,
     setInputVarValues,
     runResult,
-    availableDependencies,
-    allowDependencies,
-    handleAddDependency,
-    handleRemoveDependency,
-    handleChangeDependency,
   }
 }