Bump pip from 23.0.1 to 23.3

.circleci/gitlab-ci-env-variables.sh

@@ -2,6 +2,7 @@ echo 'export CI_BUILD_ID="$CIRCLE_BUILD_NUM"' >> $BASH_ENV
 echo 'export CI_PIPELINE_ID="$CIRCLE_WORKFLOW_ID"' >> $BASH_ENV
 echo 'export CI_COMMIT_BRANCH="$CIRCLE_BRANCH"' >> $BASH_ENV
 echo 'export ARTIFACTS_FOLDER=/home/circleci/project/artifacts' >> $BASH_ENV
+echo 'export PIPELINE_JOBS_FOLDER=/home/circleci/project/pipeline_jobs_folder' >> $BASH_ENV
 echo 'export CI_COMMIT_SHA="$CIRCLE_SHA1"' >> $BASH_ENV
 echo 'export CI_JOB_URL="$CIRCLE_BUILD_URL"' >> $BASH_ENV
 echo 'export CI_JOB_NAME="$CIRCLE_JOB"' >> $BASH_ENV

.devcontainer/Dockerfile

@@ -1,34 +1,35 @@
 FROM python:3.10-slim-bullseye
 
-ADD createCerts.sh .
-RUN apt-get update && apt-get install dos2unix -y \
-    && dos2unix /createCerts.sh \
-    && chmod +x /createCerts.sh \ 
-    && /createCerts.sh /usr/local/share/ca-certificates/certs.crt \
-    && update-ca-certificates \
-    && apt-get install python2 -y
-
 ENV USERNAME demisto
 ENV HOME /home/$USERNAME
 ENV NODE_EXTRA_CA_CERTS /usr/local/share/ca-certificates/certs.crt
 ENV PATH $PATH:$HOME/.local/bin:/root/.local/bin:/usr/local/share/nvm/current/bin
 
-# This is a workaround for VSCode devcontainer features in self signed certificate
-
-ADD https://raw.githubusercontent.com/devcontainers/features/main/src/common-utils/install.sh /install-common.sh
-ADD https://raw.githubusercontent.com/devcontainers/features/main/src/git/install.sh /install-git.sh
-ADD https://raw.githubusercontent.com/devcontainers/features/main/src/docker-in-docker/install.sh /install-dind.sh
-ADD https://raw.githubusercontent.com/devcontainers/features/main/src/node/install.sh /install-node.sh
+ADD createCerts.sh .
+RUN apt-get update && apt-get install dos2unix git python2 curl -y \
+    && dos2unix /createCerts.sh \
+    && chmod +x /createCerts.sh \ 
+    && /createCerts.sh $NODE_EXTRA_CA_CERTS \
+    && update-ca-certificates \
+    && rm -rf /features \
+    && git clone https://github.com/devcontainers/features.git /features \
+    && cd /features \
+    # locking to the latest master commit in this repo (https://github.com/devcontainers/features.git) to prevent breaking changes
+    # We should update this commit hash from time to time to
+    && git checkout 96bff0097028001e6e4126c5528d37cb8c13e785
 
-RUN UID="1000" GID="1000" bash install-common.sh 
-RUN VERSION="os-provided" bash install-git.sh
-RUN VERSION="latest" bash install-dind.sh
-RUN VERSION="lts" bash install-node.sh
+# This is a workaround for VSCode devcontainer features in self signed certificate
+RUN UID="1000" GID="1000" bash /features/src/common-utils/install.sh
+RUN VERSION="os-provided" bash /features/src/git/install.sh
+RUN VERSION="latest" bash /features/src/docker-in-docker/install.sh
+RUN VERSION="lts" bash /features/src/node/install.sh
+RUN bash /features/src/sshd/install.sh
+RUN bash /features/src/github-cli/install.sh
 
 # install poetry, configure certificate for git and tools for oh my zsh
-RUN curl -sSL https://install.python-poetry.org | python3 - \
-    && python -m pip install --user pipx \
+RUN python -m pip install --user pipx \
     && python -m pipx ensurepath \
+    && pipx install poetry \
     && git clone https://github.com/zsh-users/zsh-syntax-highlighting.git $HOME/.zsh/zsh-syntax-highlighting \
     && echo "source ~/.zsh/zsh-syntax-highlighting/zsh-syntax-highlighting.zsh" >> $HOME/.zshrc \
     && git clone https://github.com/zsh-users/zsh-autosuggestions $HOME/.zsh/zsh-autosuggestions \

.devcontainer/createCerts.sh

@@ -5,6 +5,13 @@
 # It tries to connect to a server (for instance, github), show the local certificates and save it to a file.
 
 # We connect to a random server and not paloaltonetworks.com to get external certificates. 
+# if command fails 
+
+if curl -I https://github.com > /dev/null; then
+    echo "No need to update certificate"
+    exit
+fi
+
 CONNECT_SERVER="github.com:443"
 
 FILE=$1
@@ -21,3 +28,6 @@ if [ ! -f "$FILE" ]; then
     exit
 fi
 
+git config --system http.sslCAInfo $FILE
+
+

.devcontainer/createCommand.sh

@@ -4,24 +4,26 @@ set -e
 
 echo "Fixing permissions"
 
-sudo chown demisto .venv
-sudo chown demisto node_modules
-sudo chown -R demisto $HOME
-sudo chown -R demisto /workspaces
-
-echo "Setting up git certificate"
+sudo chown demisto /workspaces /workspaces/content
+sudo chown -R demisto /workspaces/content/.vscode /workspaces/content/.git /workspaces/content/.venv /workspaces/content/node_modules /workspaces/content/package-lock.json
 
-sudo git config --system http.sslCAInfo /usr/local/share/ca-certificates/certs.crt
+sudo chown -R demisto $HOME
 
 echo "Setting up VSCode paths"
 
 cp .devcontainer/settings.json .vscode/settings.json 
 touch CommonServerUserPython.py
+touch DemistoClassApiModule.py
 path=$(printf '%s:' Packs/ApiModules/Scripts/*)
 rm -f .env
 echo "PYTHONPATH=""$path"":$PYTHONPATH" >> .env
 echo "MYPYPATH=""$path"":$MYPYPATH" >> .env
 
+echo "Setting up git safe directory"
+git config --global --add safe.directory /workspaces/content
+
 echo "Setting up content dependencies"
+.hooks/bootstrap
 
-NO_HOOKS=1 .hooks/bootstrap
+echo "Run demisto-sdk pre-commit to cache dependencies"
+poetry run demisto-sdk pre-commit >/dev/null 2>&1 || true

.devcontainer/devcontainer.json

@@ -1,3 +1,4 @@
+// Development container for Content.
 {
 	"name": "XSOAR Content",
 	"build": {
@@ -14,52 +15,11 @@
 		"source=node-modules,target=${containerWorkspaceFolder}/node_modules,type=volume",
 		"source=dind-var-lib-docker,target=/var/lib/docker,type=volume"
 	],
+	"containerUser": "demisto",
 	"remoteUser": "demisto",
-	"settings": {
-		"terminal.integrated.defaultProfile.linux": "zsh",
-		"terminal.integrated.profiles.linux": {
-			"zsh": {
-				"path": "/bin/zsh"
-			}
-		},
-		"cSpell.words": [
-			"demisto",
-			"xsoar",
-			"xsiam",
-			"fromversion",
-			"toversion",
-			"marketplacev",
-			"ciac",
-			"whois",
-		]
-	},
-	"extensions": [
-		"cortexxsoarext.xsoar",
-		"ms-python.python",
-		"ms-vscode.PowerShell",
-		"ms-toolsai.jupyter",
-		"timonwong.shellcheck",
-		"GitHub.vscode-pull-request-github",
-		"eamodio.gitlens",
-		"ms-azuretools.vscode-docker",
-		"ms-vscode-remote.remote-containers",
-		"streetsidesoftware.code-spell-checker",
-		"njpwerner.autodocstring",
-		"VisualStudioExptTeam.vscodeintellicode",
-		"yzhang.markdown-all-in-one",
-		"shd101wyy.markdown-preview-enhanced",
-		"Gruntfuggly.todo-tree",
-		"redhat.vscode-yaml",
-		"PKief.material-icon-theme",
-		"mikestead.dotenv",
-		"KevinRose.vsc-python-indent",
-		"rangav.vscode-thunder-client",
-		"ms-python.black-formatter",
-		"LittleFoxTeam.vscode-python-test-adapter"
-	],
 	"remoteEnv": {
 		"LOCAL_WORKSPACE_PATH": "${localWorkspaceFolder}",
-		"PATH": "${containerWorkspaceFolder}/.venv/bin:${containerEnv:PATH}",
+		"PATH": "${containerEnv:PATH}:${containerWorkspaceFolder}/.venv/bin",
 		"CONTENT": "${containerWorkspaceFolder}",
 		"DEMISTO_SDK_CONTENT_PATH": "${containerWorkspaceFolder}",
 		"PYTHONPATH": "${containerWorkspaceFolder}:${containerWorkspaceFolder}/Packs/Base/Scripts/CommonServerPython/:${containerWorkspaceFolder}/Packs/Base/Scripts/CommonServerUserPython/:${containerWorkspaceFolder}/Tests/demistomock/:${containerEnv:PYTHONPATH}",
@@ -70,26 +30,77 @@
 		"DEMISTO_VERIFY_SSL": "${localEnv:DEMISTO_VERIFY_SSL}",
 		"DEMISTO_API_KEY": "${localEnv:DEMISTO_API_KEY}"
 	},
-	// this is commented out until VSCode will fix self signed certificate issues
-	// "features": {
-	// 	"ghcr.io/devcontainers/features/docker-in-docker:1": {
-	// 		"version": "latest"
-	// 	},
-	// 	"ghcr.io/devcontainers/features/git:1": {
-	// 		"version": "os-provided"
-	// 	},
-	// 	"ghcr.io/devcontainers/features/powershell:1": {
-	// 		"version": "latest"
-	// 	},
-	// 	"ghcr.io/devcontainers/features/common-utils:1": {
-	// 		"version": "latest"
-	// 	},
-	// 	"ghcr.io/devcontainers/features/node:1": {
-	// 		"version": "latest"
-	// 	}
-	// },
-	// "overrideFeatureInstallOrder": [
-	// 	"ghcr.io/devcontainers/features/common-utils:1"
-	// ],
-	"onCreateCommand": "dos2unix .devcontainer/createCommand.sh && chmod +x .devcontainer/createCommand.sh && bash .devcontainer/createCommand.sh",
+	"customizations": {
+		"vscode": {
+			"settings": {
+				"terminal.integrated.defaultProfile.linux": "zsh",
+				"terminal.integrated.profiles.linux": {
+					"zsh": {
+						"path": "/bin/zsh"
+					}
+				},
+				"cSpell.words": [
+					"demisto",
+					"xsoar",
+					"xsiam",
+					"fromversion",
+					"toversion",
+					"marketplacev",
+					"ciac",
+					"whois",
+				]
+			},
+			"extensions": [
+				"cortexxsoarext.xsoar",
+				"GitLab.gitlab-workflow",
+				"ms-python.python",
+				"GitHub.vscode-pull-request-github",
+				"eamodio.gitlens",
+				"ms-azuretools.vscode-docker",
+				"ms-vscode-remote.remote-containers",
+				"streetsidesoftware.code-spell-checker",
+				"njpwerner.autodocstring",
+				"VisualStudioExptTeam.vscodeintellicode",
+				"redhat.vscode-yaml",
+				"PKief.material-icon-theme",
+				"mikestead.dotenv",
+				"KevinRose.vsc-python-indent",
+				"LittleFoxTeam.vscode-python-test-adapter",
+				"charliermarsh.ruff",
+				"ryanluker.vscode-coverage-gutters",
+				"ms-python.debugpy",
+				"EditorConfig.EditorConfig",
+				"DavidAnson.vscode-markdownlint",
+				"ms-vscode.powershell"
+			]
+		}
+	},
+// this is commented out until VSCode will fix self signed certificate issues
+// "features": {
+// 	"ghcr.io/devcontainers/features/docker-in-docker:1": {
+// 		"version": "latest"
+// 	},
+// 	"ghcr.io/devcontainers/features/git:1": {
+// 		"version": "os-provided"
+// 	},
+// 	"ghcr.io/devcontainers/features/powershell:1": {
+// 		"version": "latest"
+// 	},
+// 	"ghcr.io/devcontainers/features/common-utils:1": {
+// 		"version": "latest"
+// 	},
+// 	"ghcr.io/devcontainers/features/node:1": {
+// 		"version": "latest"
+// 	}
+// },
+// "overrideFeatureInstallOrder": [
+// 	"ghcr.io/devcontainers/features/common-utils:1"
+// ],
+"onCreateCommand": "sudo dos2unix -n .devcontainer/createCommand.sh .devcontainer/createCommand_unix.sh && bash .devcontainer/createCommand_unix.sh",
+"postStartCommand": "poetry install && poetry run demisto-sdk pre-commit >/dev/null 2>&1 || true",
+"hostRequirements": {
+	"cpus": 4,
+	"memory": "8gb",
+	"storage": "32gb"
+ }
 }

.devcontainer/settings.json

@@ -1,8 +1,8 @@
 {
     "python.defaultInterpreterPath": "${workspaceFolder}/.venv/bin/python",
     "python.linting.enabled": true,
-    "python.linting.flake8Enabled": true,
     "python.linting.mypyEnabled": true,
+    "python.linting.flake8Enabled": false,
     "python.linting.mypyArgs": [
         "--follow-imports=silent",
         "--ignore-missing-imports",

.editorconfig

@@ -0,0 +1,23 @@
+root = true
+
+[*]
+charset = utf-8
+end_of_line = lf
+indent_size = 4
+indent_style = space
+insert_final_newline = false
+tab_width = 4
+
+[{*.bash,*.sh,*.zsh}]
+indent_size = 2
+tab_width = 2
+
+[{*.py,*.pyw}]
+max_line_length = 130
+
+[{*.yaml,*.yml}]
+indent_size = 2
+max_line_length = off
+
+[{*.md,*.markdown}]
+max_line_length = off