Initial commit

KOLANICH · KOLANICH · commit 0f6f3240f974 · 2023-03-11T13:43:09.000+03:00
diff --git a/.ci/aptPackagesToInstall.txt b/.ci/aptPackagesToInstall.txt
diff --git a/.ci/pythonPackagesToInstallFromGit.txt b/.ci/pythonPackagesToInstallFromGit.txt
diff --git a/.editorconfig b/.editorconfig
@@ -0,0 +1,12 @@
+root = true
+
+[*]
+charset = utf-8
+indent_style = tab
+indent_size = 4
+insert_final_newline = true
+end_of_line = lf
+
+[*.{yml,yaml}]
+indent_style = space
+indent_size = 2
diff --git a/.github/.templateMarker b/.github/.templateMarker
@@ -0,0 +1 @@
+KOLANICH/python_project_boilerplate.py
diff --git a/.github/dependabot.yml b/.github/dependabot.yml
@@ -0,0 +1,8 @@
+version: 2
+updates:
+  - package-ecosystem: "pip"
+    directory: "/"
+    schedule:
+      interval: "daily"
+    allow:
+      - dependency-type: "all"
diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml
@@ -0,0 +1,15 @@
+name: CI
+on:
+  push:
+    branches: [master]
+  pull_request:
+    branches: [master]
+
+jobs:
+  build:
+    runs-on: ubuntu-22.04
+    steps:
+      - name: typical python workflow
+        uses: KOLANICH-GHActions/typical-python-workflow@master
+        with:
+          github_token: ${{ secrets.GITHUB_TOKEN }}
diff --git a/.gitignore b/.gitignore
@@ -0,0 +1,10 @@
+__pycache__
+*.py[co]
+/*.egg-info
+*.srctrlbm
+*.srctrldb
+build
+dist
+.eggs
+monkeytype.sqlite3
+/.ipynb_checkpoints
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
@@ -0,0 +1,51 @@
+image: registry.gitlab.com/kolanich-subgroups/docker-images/fixed_python:latest
+
+variables:
+  DOCKER_DRIVER: overlay2
+  SAST_ANALYZER_IMAGE_TAG: latest
+  SAST_DISABLE_DIND: "true"
+  SAST_CONFIDENCE_LEVEL: 5
+  CODECLIMATE_VERSION: latest
+
+include:
+  - template: SAST.gitlab-ci.yml
+  - template: Code-Quality.gitlab-ci.yml
+  - template: License-Management.gitlab-ci.yml
+
+build:
+  tags:
+    - shared
+    - linux
+  stage: build
+  variables:
+    GIT_DEPTH: "1"
+    PYTHONUSERBASE: ${CI_PROJECT_DIR}/python_user_packages
+
+  before_script:
+    - export PATH="$PATH:$PYTHONUSERBASE/bin" # don't move into `variables`
+    - apt-get update
+    # todo:
+    #- apt-get -y install 
+    #- pip3 install --upgrade 
+    #- python3 ./fix_python_modules_paths.py
+
+  script:
+    - python3 -m build -nw bdist_wheel
+    - mv ./dist/*.whl ./dist/GitLabInstancesDataset-0.CI-py3-none-any.whl
+    - pip3 install --upgrade ./dist/*.whl
+    - coverage run --source=GitLabInstancesDataset -m --branch pytest --junitxml=./rspec.xml ./tests/test.py
+    - coverage report -m
+    - coverage xml
+
+  coverage: "/^TOTAL(?:\\s+\\d+){4}\\s+(\\d+%).+/"
+
+  cache:
+    paths:
+      - $PYTHONUSERBASE
+
+  artifacts:
+    paths:
+      - dist
+    reports:
+      junit: ./rspec.xml
+      cobertura: ./coverage.xml
diff --git a/Code_Of_Conduct.md b/Code_Of_Conduct.md
@@ -0,0 +1 @@
+No codes of conduct!
diff --git a/GitLabInstancesDataset/KnownGitLabInstances.txt b/GitLabInstancesDataset/KnownGitLabInstances.txt
@@ -0,0 +1,56 @@
+ais.matadorsuite.com
+altitude.otago.ac.nz
+bsvgit.informatik.uni-leipzig.de
+code.ach.gov.ru
+code.datasciencedojo.com
+code.foxkit.us
+code.ornl.gov
+code.videolan.org
+cseegit.essex.ac.uk
+dev.seemoo.tu-darmstadt.de
+forge.etsi.org
+foss.heptapod.net
+framagit.org
+geo1.harishsr.com
+g.info.gov.ru
+git.amrita.edu
+git.app.uib.no
+git.cas.unt.edu
+git-ce.rwth-aachen.de
+git.circ.rochester.edu
+git.congatec.com
+git.fim.uni-passau.de
+gitgud.io
+git.global-system.ru
+git.gutta.com
+githacks.org
+git.ieee.uni-passau.de
+git.iem.at
+git.imp.fu-berlin.de
+git.ispconfig.org
+git.jinr.ru
+git.mbk-lab.ru
+git.metabarcoding.org
+git.miem.hse.ru
+git.oreolek.ru
+git.phys.ucalgary.ca
+git.rwth-aachen.de
+git.sakha.gov.ru
+git.scc.kit.edu
+git.seco.com
+git.soton.ac.uk
+git.uark.edu
+git.uclalemur.com
+git.ucsc.edu
+git.unl.edu
+git.uwaterloo.ca
+git.adelielinux.org
+code.foxkit.us
+invent.kde.org
+lobogit.unm.edu
+mt11.net.ru
+production.eng.it
+salsa.debian.org
+selma.cise.ufl.edu
+version.helsinki.fi
+www.rm.7733.ru
diff --git a/GitLabInstancesDataset/__init__.py b/GitLabInstancesDataset/__init__.py
@@ -0,0 +1,24 @@
+import typing
+from pathlib import Path
+
+__all__ = ("isGitLab",)
+
+
+def _convertToTuple(s: str) -> typing.Tuple[str]:
+	return tuple(el.lower() for el in reversed(s.split(".")) if el)
+
+
+knownServicesFile = Path(__file__).parent.absolute() / "KnownGitLabInstances.txt"
+knownServices = set(_convertToTuple(el) for el in knownServicesFile.read_text().splitlines())
+
+
+def isGitLab(domain: typing.Union[str, tuple]) -> bool:
+	"""Pass a domain name NOT CONTAINING the token `gitlab`, and get a boolean saying if it is present within the dataset"""
+
+	if isinstance(domain, str):
+		domain = _convertToTuple(domain)
+
+	if "gitlab" in domain:
+		return True
+
+	return domain in knownServices
diff --git a/MANIFEST.in b/MANIFEST.in
@@ -0,0 +1,4 @@
+include UNLICENSE
+include *.md
+include tests
+include .editorconfig
diff --git a/ReadMe.md b/ReadMe.md
@@ -0,0 +1,35 @@
+GitLabInstancesDataset.py [![Unlicensed work](https://raw.githubusercontent.com/unlicense/unlicense.org/master/static/favicon.png)](https://unlicense.org/)
+==========================
+~~[wheel (GitLab)](https://gitlab.com/prebuilder/GitLabInstancesDataset.py/-/jobs/artifacts/master/raw/dist/GitLabInstancesDataset-0.CI-py3-none-any.whl?job=build)~~
+[wheel (GHA via `nightly.link`)](https://nightly.link/prebuilder/GitLabInstancesDataset.py/workflows/CI/master/GitLabInstancesDataset-0.CI-py3-none-any.whl)
+~~![GitLab Build Status](https://gitlab.com/prebuilder/GitLabInstancesDataset.py/badges/master/pipeline.svg)~~
+~~![GitLab Coverage](https://gitlab.com/prebuilder/GitLabInstancesDataset.py/badges/master/coverage.svg)~~
+[![GitHub Actions](https://github.com/prebuilder/GitLabInstancesDataset.py/workflows/CI/badge.svg)](https://github.com/prebuilder/GitLabInstancesDataset.py/actions/)
+![N∅ hard dependencies](https://shields.io/badge/-N∅_Ъ_deps!-0F0)
+[![Libraries.io Status](https://img.shields.io/librariesio/github/prebuilder/GitLabInstancesDataset.py.svg)](https://libraries.io/github/prebuilder/GitLabInstancesDataset.py)
+[![Code style: antiflash](https://img.shields.io/badge/code%20style-antiflash-FFF.svg)](https://github.com/KOLANICH-tools/antiflash.py)
+
+A dataset of standalone GitLab instances to determine if the URI is hosten on GitLab without probing it.
+
+While this is a python package, the actual version of the dataset in `txt` format can be downloaded by the URI https://raw.githubusercontent.com/prebuilder/GitLabInstancesDataset/master/GitLabInstancesDataset/KnownGitLabInstances.txt and used from any language you like.
+
+# How to used
+
+1. Parse URI, extract its domain.
+2. Check if we can guess if the service is `GitLab` from its domain name or URI only. If we can, it is not to be included into the dataset.
+	a. Check if the domain contains the substring `gitlab`. 
+	b. Check if the path contains the substring `gitlab`. 
+3. Check the name againsrt the dataset. In python it can be done using `isGitLab(domainName)` function.
+
+# Inclusion criteria
+
+* Neither domain name nor the path to the actual service contains the substring `gitlab`.
+* The dataset contains **domain names** only. Don't send URIs here.
+* The domain names in the list must be
+	* normalized to lower case
+	* unique
+	* sorted
+	* must not contain any empty components
+
+* LF line ending
+* no IDNs currently
diff --git a/UNLICENSE b/UNLICENSE
@@ -0,0 +1,24 @@
+This is free and unencumbered software released into the public domain.
+
+Anyone is free to copy, modify, publish, use, compile, sell, or
+distribute this software, either in source code form or as a compiled
+binary, for any purpose, commercial or non-commercial, and by any
+means.
+
+In jurisdictions that recognize copyright laws, the author or authors
+of this software dedicate any and all copyright interest in the
+software to the public domain. We make this dedication for the benefit
+of the public at large and to the detriment of our heirs and
+successors. We intend this dedication to be an overt act of
+relinquishment in perpetuity of all present and future rights to this
+software under copyright law.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR
+OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+OTHER DEALINGS IN THE SOFTWARE.
+
+For more information, please refer to <https://unlicense.org/>
diff --git a/pyproject.toml b/pyproject.toml
@@ -0,0 +1,35 @@
+[build-system]
+requires = ["setuptools>=61.2.0", "wheel", "setuptools_scm[toml]>=3.4.3"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "GitLabInstancesDataset"
+readme = "ReadMe.md"
+description = "A dataset of standalone GitLab instances to determine if the URI is hosten on GitLab without probing it."
+authors = [{name = "KOLANICH"}]
+classifiers = [
+	"Development Status :: 4 - Beta",
+	"Environment :: Other Environment",
+	"Intended Audience :: Developers",
+	"License :: Public Domain",
+	"Operating System :: OS Independent",
+	"Programming Language :: Python",
+	"Programming Language :: Python :: 3",
+	"Programming Language :: Python :: 3 :: Only",
+	"Topic :: Software Development :: Libraries :: Python Modules",
+]
+keywords = ["GitLab", "dataset"]
+license = {text = "Unlicense"}
+requires-python = ">=3.4"
+dynamic = ["version"]
+
+[project.urls]
+Homepage = "https://github.com/prebuilder/GitLabInstancesDataset.py"
+
+[tool.setuptools]
+zip-safe = true
+
+[tool.setuptools.packages.find]
+exclude = ["tests"]
+
+[tool.setuptools_scm]
diff --git a/tests/tests.py b/tests/tests.py
@@ -0,0 +1,28 @@
+#!/usr/bin/env python3
+import sys
+from pathlib import Path
+import unittest
+
+sys.path.insert(0, str(Path(__file__).parent.parent))
+
+from collections import OrderedDict
+
+dict = OrderedDict
+
+import GitLabInstancesDataset
+from GitLabInstancesDataset import isGitLab, _convertToTuple
+
+
+class Tests(unittest.TestCase):
+	def testDomainNameNormalization(self):
+		self.assertEqual(("ni", "said", "who", "knight"), _convertToTuple("knigHt.wHo.saId.Ni"))
+
+	def testContainsGitLab(self):
+		self.assertFalse(isGitLab("gitlab.com"))
+
+	def testInSet(self):
+		self.assertTrue(isGitLab("salsa.debian.org"))
+
+
+if __name__ == "__main__":
+	unittest.main()
