Initial commit

KOLANICH · KOLANICH · commit de1bf540aa8a · 2023-02-24T00:29:38.000+03:00
diff --git a/.ci/before.sh b/.ci/before.sh
@@ -0,0 +1,4 @@
+git clone --depth=1 https://gitlab.com/kaitaiStructCompile.py/kaitaiStructCompile.py.git
+export KSCP=./kaitaiStructCompile.py
+git clone --depth=1 https://gitlab.com/kaitaiStructCompile.py/kaitaiStructCompile.tests.ksys $KSCP/tests/ksys
+pip install --upgrade $KSCP
diff --git a/.editorconfig b/.editorconfig
@@ -0,0 +1,12 @@
+root = true
+
+[*]
+charset = utf-8
+indent_style = tab
+indent_size = 4
+insert_final_newline = true
+end_of_line = lf
+
+[*.{yml,yaml,ksy}]
+indent_style = space
+indent_size = 2
diff --git a/.github/.templateMarker b/.github/.templateMarker
@@ -0,0 +1 @@
+KOLANICH/python_project_boilerplate.py
diff --git a/.github/dependabot.yml b/.github/dependabot.yml
@@ -0,0 +1,8 @@
+version: 2
+updates:
+  - package-ecosystem: "pip"
+    directory: "/"
+    schedule:
+      interval: "daily"
+    allow:
+      - dependency-type: "all"
diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml
@@ -0,0 +1,15 @@
+name: CI
+on:
+  push:
+    branches: [master]
+  pull_request:
+    branches: [master]
+
+jobs:
+  build:
+    runs-on: ubuntu-22.04
+    steps:
+      - name: typical python workflow
+        uses: KOLANICH-GHActions/typical-python-workflow@master
+        with:
+          github_token: ${{ secrets.GITHUB_TOKEN }}
diff --git a/.gitignore b/.gitignore
@@ -0,0 +1,9 @@
+__pycache__
+*.pyc
+*.pyo
+/kaitaiStructCompile.egg-info
+/build
+/dist
+/.eggs
+/tests/output
+/tests/formats
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
@@ -0,0 +1,50 @@
+image: registry.gitlab.com/kaitaistructcompile.py/kaitai_struct_python_docker:latest
+
+stages:
+  - build
+  - test
+
+variables:
+  GIT_DEPTH: "1"
+  DOCKER_DRIVER: overlay2
+  SAST_ANALYZER_IMAGE_TAG: latest
+  SAST_DISABLE_DIND: "true"
+  SAST_CONFIDENCE_LEVEL: 5
+  CODECLIMATE_VERSION: latest
+
+include:
+  - template: SAST.gitlab-ci.yml
+  - template: Code-Quality.gitlab-ci.yml
+
+
+build:
+  tags:
+    - shared
+    - linux
+  stage: build
+  variables:
+    PYTHONUSERBASE: ${CI_PROJECT_DIR}/python_user_packages
+    KAITAI_STRUCT_COMPILE_FORCE_BACKEND: CLI
+
+  before_script:
+    - export PYTHON_MODULES_DIR=${PYTHONUSERBASE}/lib/python3.7/site-packages
+    - export EXECUTABLE_DEPENDENCIES_DIR=${PYTHONUSERBASE}/bin
+    - export PATH="$PATH:$EXECUTABLE_DEPENDENCIES_DIR" # don't move into `variables` any of them, it is unordered
+    - source ./.ci/before.sh
+
+  script:
+    - python3 ./setup.py bdist_wheel
+    - pip3 install --upgrade ./dist/*.whl
+    - coverage run --branch --source=kaitaiStructCompile -m pytest --junitxml=./rspec.xml $KSCP/tests/test.py
+    - mkdir ./wheels
+    - mv ./dist/*.whl ./wheels/kaitaiStructCompile.backend.CLI-0.CI-py3-none-any.whl
+
+  cache:
+    paths:
+      - /lib/python3*/site-packages
+      - tests/kaitai_struct_formats
+
+  artifacts:
+    paths:
+      - wheels
+      - tests/output
diff --git a/Code_Of_Conduct.md b/Code_Of_Conduct.md
@@ -0,0 +1 @@
+No codes of conduct!
diff --git a/MANIFEST.in b/MANIFEST.in
@@ -0,0 +1,5 @@
+include UNLICENSE
+include *.md
+include tests
+include .editorconfig
+global-include *.json
diff --git a/ReadMe.md b/ReadMe.md
@@ -0,0 +1,18 @@
+kaitaiStructCompile.py CLI backend [![Unlicensed work](https://raw.githubusercontent.com/unlicense/unlicense.org/master/static/favicon.png)](https://unlicense.org/)
+==================================
+[![GitLab build status](https://gitlab.com/kaitaiStructCompile.py/kaitaiStructCompile.backend.CLI/badges/master/pipeline.svg)](https://gitlab.com/kaitaiStructCompile.py/kaitaiStructCompile.backend.CLI/commits/master)
+[![GitLab coverage](https://gitlab.com/kaitaiStructCompile.py/kaitaiStructCompile.backend.CLI/badges/master/coverage.svg)](https://gitlab.com/kaitaiStructCompile.py/kaitaiStructCompile.backend.CLI/commits/master)
+[![Code style: antiflash](https://img.shields.io/badge/code%20style-antiflash-FFF.svg)](https://github.com/KOLANICH-tools/antiflash.py)
+
+This is a CLI backend for [`kaitaiStructCompile.py`](https://gitlab.com/kaitaiStructCompile.py/kaitaiStructCompile).
+
+Cons:
+* insecure: uses subprocess.call, passes arguments via a command line; also races on access to files are possible: if an attacker wrote the temporary file between `ksc` have it unlocked and the app have not yet read, he can replace the file contents. If `importer` is used, it is code execution and injection attack.
+* slow: creates subprocesses, interacts via on-disk files and stdout;
+* burns in SSD - since it uses temporary files.
+
+
+Pros:
+* universal - should work on any OS
+* requires no dependencies
+* permissive license
diff --git a/UNLICENSE b/UNLICENSE
@@ -0,0 +1,24 @@
+This is free and unencumbered software released into the public domain.
+
+Anyone is free to copy, modify, publish, use, compile, sell, or
+distribute this software, either in source code form or as a compiled
+binary, for any purpose, commercial or non-commercial, and by any
+means.
+
+In jurisdictions that recognize copyright laws, the author or authors
+of this software dedicate any and all copyright interest in the
+software to the public domain. We make this dedication for the benefit
+of the public at large and to the detriment of our heirs and
+successors. We intend this dedication to be an overt act of
+relinquishment in perpetuity of all present and future rights to this
+software under copyright law.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR
+OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+OTHER DEALINGS IN THE SOFTWARE.
+
+For more information, please refer to <https://unlicense.org/>
diff --git a/kaitaiStructCompileBackendCLI/__init__.py b/kaitaiStructCompileBackendCLI/__init__.py
@@ -0,0 +1,212 @@
+import io
+import os
+import subprocess
+import sys
+import tempfile
+import typing
+from functools import wraps
+from pathlib import Path
+
+
+class Flag:
+	slots = ("value",)
+
+	def __init__(self, value):
+		self.value = value
+
+	def __hash__(self):
+		return hash(self.value)
+
+	def __bool__(self):
+		return self.value
+
+	def __repr__(self):
+		return self.__class__.__name__ + "(" + repr(self.value) + ")"
+
+
+langNamespaceCliArgMapping = {
+	"python": "python-package",
+	"go": "go-package",
+	"jre": "java-package",
+	"dotNet": "dotnet-namespace",
+	"php": "php-namespace",
+	"cpp": "cpp-namespace",
+	"nim": "nim-module"
+}
+
+
+class paramsRemapping:
+	def verbose(v):
+		if v:
+			return {"--verbose": ",".join(v)}
+		else:
+			return {}
+
+	def opaqueTypes(v):
+		return {"--opaque-types": str(v).lower()}
+
+	def autoRead(v):
+		return {"--no-auto-read": Flag(not v)}
+
+	def readStoresPos(v):
+		return {"--read-pos": Flag(v)}
+
+	##########
+
+	def namespaces(v):
+		res = {}
+		for langName, v in v.items():
+			if v is not None:
+				res["--" + langNamespaceCliArgMapping[langName]] = v
+		return res
+
+	def target(v):
+		return {"--target": v}
+
+	def destDir(v):
+		return {"--outdir": str(v)}
+
+	def additionalFlags(v):
+		if isinstance(v, (list, tuple)):
+			f = Flag(True)
+			return {kk: f for kk in v}
+		else:
+			res = {}
+			for vk, vv in v.items():
+				if hasattr(paramsRemapping, vk):
+					res.update(getattr(paramsRemapping, vk)(vv))
+				else:
+					if vv is None:
+						vv = Flag(True)
+					res[vk] = vv
+			print("additionalFlags res", res)
+			return res
+
+	def importPath(v):
+		return {"--import-path": str(v)}
+
+
+def init(ICompilerModule, KaitaiCompilerException, utils, defaults):
+
+	osBinariesNamesTable = {
+		"nt": (defaults.compilerName + ".bat"),
+		"posix": defaults.compilerName
+	}
+
+	class CLIPrefsStorage(ICompilerModule.IPrefsStorage):
+		@wraps(ICompilerModule.IPrefsStorage.__init__)
+		def __init__(self, **kwargs):
+			self._stor = {}
+			for argName, v in kwargs.items():
+				if v is not None:
+					if hasattr(paramsRemapping, argName):
+						f = getattr(paramsRemapping, argName)
+						self._stor.update(f(v))
+					else:
+						raise ValueError("`" + argName + "` is not an implemented arg")
+
+		def __iadd__(self, other):
+			self._stor.update(other._stor)
+			return self
+
+		def __add__(self, other):
+			res = self.__class__()
+			res += self
+			res += other
+			return res
+
+		def __repr__(self):
+			return self.__class__.__name__ + "<" + " ".join(self()) + ">"
+
+		def __call__(self):
+			params = []
+			for k, v in self._stor.items():
+				if v is not None:
+					if not isinstance(v, Flag):
+						params.append(k)
+						params.append(v)
+					else:
+						if v:
+							params.append(k)
+
+			return params
+
+	class CLICompiler(ICompilerModule.ICompiler):
+		def __init__(self, progressCallback=None, dirs=None, namespaces=None, additionalFlags: typing.Iterable[str] = (), importPath=None, verbose=(), opaqueTypes=None, autoRead: bool = None, readStoresPos: bool = None, target: str = "python", **kwargs):
+			super().__init__(progressCallback, dirs)
+
+			self.compilerExecutable = self.dirs.bin / osBinariesNamesTable[os.name]
+
+			self.commonFlags = CLIPrefsStorage(additionalFlags=("--ksc-json-output",))  # flags needed for this class to work correctly, though a user is allowed to redefine
+
+			self.commonFlags += CLIPrefsStorage(namespaces=namespaces, additionalFlags=additionalFlags, importPath=importPath, verbose=verbose, opaqueTypes=opaqueTypes, autoRead=autoRead, readStoresPos=readStoresPos, **kwargs)
+
+			if not self.compilerExecutable.exists():
+				raise KaitaiCompilerException("Compiler executable " + str(self.compilerExecutable) + " doesn't exist")
+
+		def compile_(self, sourceFilesAbsPaths: typing.Iterable[Path], destDir: Path, additionalFlags: typing.Iterable[str], verbose, opaqueTypes, autoRead: bool, readStoresPos: bool, target: str = "python", needInMemory: bool = False, **kwargs) -> typing.Mapping[str, ICompilerModule.ICompileResult]:
+			if destDir is None:
+				with tempfile.TemporaryDirectory() as tmpDir:
+					return self.compile__(sourceFilesAbsPaths, Path(tmpDir), additionalFlags, verbose, opaqueTypes, autoRead, readStoresPos, target, needInMemory, **kwargs)
+			else:
+				return self.compile__(sourceFilesAbsPaths, destDir, additionalFlags, verbose, opaqueTypes, autoRead, readStoresPos, target, needInMemory, **kwargs)
+
+		def compile__(self, sourceFilesAbsPaths: typing.Iterable[Path], destDir: Path, additionalFlags: typing.Iterable[str], verbose, opaqueTypes, autoRead: bool, readStoresPos: bool, target: str = "python", needInMemory: bool = False, **kwargs) -> typing.Mapping[str, ICompilerModule.ICompileResult]:
+			"""Compiles KS package with kaitai-struct-compiler"""
+			print("commonFlags", self.commonFlags)
+			params = [str(self.compilerExecutable)]
+
+			params += (self.commonFlags + CLIPrefsStorage(destDir=destDir, additionalFlags=additionalFlags, verbose=verbose, opaqueTypes=opaqueTypes, autoRead=autoRead, readStoresPos=readStoresPos, target=target))()
+
+			params.extend((str(p) for p in sourceFilesAbsPaths))
+
+			with subprocess.Popen(params, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) as proc:
+				with io.TextIOWrapper(proc.stdout) as stdoutPipe:
+					msg = stdoutPipe.read()
+				proc.wait()
+				# print(msg, proc.returncode)
+
+				if proc.returncode or msg.find("Exception in thread") > -1:
+					raise KaitaiCompilerException(msg)
+
+			res = utils.json.loads(msg)
+			from pprint import pprint
+
+			pprint(res)
+
+			resultModules = {}
+			issues = []
+
+			for srcFile, fRes in res.items():
+				srcFile = Path(srcFile)
+
+				def processErrors(obj: dict):
+					errors = obj.get("errors", ())
+					for error in errors:
+						issues.append(ICompilerModule.issueFactory(srcFile, error.get("file", None), error.get("message", None), line=error.get("line", None), column=error.get("col", None), path=error.get("path", None)))
+
+				processErrors(fRes)
+
+				if "output" in fRes:
+					for packageName, specResult in fRes["output"][target].items():
+						processErrors(specResult)
+
+						for fSpecResultDescr in specResult.get("files", ()):
+							resultModules[packageName] = ICompilerModule.InFileCompileResult(moduleName=packageName, mainClassName=specResult["topLevelName"], msg=msg, path=destDir / fSpecResultDescr["fileName"])
+
+				firstSpecName = fRes.get("firstSpecName", None)
+				if firstSpecName is not None:
+					try:
+						resultModules[firstSpecName].sourcePath = srcFile
+					except KeyError:
+						pass
+
+			print(resultModules)
+			if issues:
+				if len(issues) == 1:
+					raise issues[0]
+
+				raise KaitaiCompilerException(issues)
+			return resultModules
+
+	return CLICompiler
diff --git a/kaitaiStructCompileBackendCLI/py.typed b/kaitaiStructCompileBackendCLI/py.typed
diff --git a/pyproject.toml b/pyproject.toml
