Use Gitlab PyPi package

lief-project · Mar 28, 2022 · cadb803 · cadb803
1 parent 2e9ed7c
commit cadb803
Show file tree

Hide file tree

Showing 9 changed files with 102 additions and 16 deletions.
diff --git a/.github/deploy.py b/.github/deploy.py
@@ -8,10 +8,15 @@
 import shutil
 import json
 import requests
+import itertools
 from collections import defaultdict
 from datetime import datetime
 from mako.template import Template
 from enum import Enum, auto
+import twine
+
+from twine.settings import Settings as TwineSettings
+from twine.commands import upload as twine_upload
 
 class CI(Enum):
     UNKNOWN        = auto()
@@ -125,7 +130,7 @@ def get_tag(ci):
 CI_PRETTY_NAME = pretty_ci_name(CURRENT_CI)
 logger.info("CI: %s", CI_PRETTY_NAME)
 
-ALLOWED_BRANCHES = {"master", "deploy", "devel"}
+ALLOWED_BRANCHES = {"master", "deploy", "devel", "feat/gitlab-packages"}
 BRANCH_NAME = get_branch(CURRENT_CI)
 TAG_NAME    = get_tag(CURRENT_CI)
 IS_TAGGED   = TAG_NAME is not None and len(TAG_NAME) > 0
@@ -178,7 +183,11 @@ def get_tag(ci):
 BUILD_DIR             = REPODIR / "build"
 
 # Gitlab
-GITLAB_PROJECT_ID = 34840840 # https://gitlab.com/lief-project/packages
+GITLAB_PROJECT_ID = int(os.getenv("GITLAB_PROJECT_ID", 0)) # https://gitlab.com/lief-project/packages
+
+if GITLAB_PROJECT_ID == 0:
+    logger.error("GITLAB_PROJECT_ID is not set!")
+    sys.exit(1)
 
 logger.debug("Working directory: %s", CI_CWD)
 
@@ -205,6 +214,27 @@ def get_tag(ci):
 </html>
 """
 
+class NotFound(Exception):
+    pass
+
+def delete_package(info):
+    logger.info("Deleting %s [%s] (%s)", info["package_type"], info["name"], info["version"])
+    url = "https://gitlab.com/api/v4/projects/{project_id}/packages/{id}"
+    headers = {
+        "PRIVATE-TOKEN": LIEF_GITLAB_TOKEN
+    }
+
+    url = url.format(
+        project_id=GITLAB_PROJECT_ID,
+        id=info["id"],
+    )
+    r = requests.delete(url, headers=headers)
+
+    if r.status_code != 204:
+        logger.error("Error while deleting the package (%s)", r.text)
+        raise NotFound
+
+
 def delete_file(file_info):
     logger.info("Deleting %s [%s]", file_info["file_name"], file_info["created_at"])
     url = "https://gitlab.com/api/v4/projects/{project_id}/packages/{pkd_id}/package_files/{file_id}"
@@ -275,6 +305,52 @@ def push_file(file_path: str, pkg_name: str, version: str):
 
     return r.json()
 
+def is_pypi(info) -> bool:
+    return info["package_type"] == "pypi"
+
+def is_generic(info) -> bool:
+    return info["package_type"] == "generic"
+
+def delete_pypi():
+    for pkg in filter(is_pypi, list_packages()):
+        try:
+            delete_package(pkg)
+        except NotFound:
+            continue
+
+def pypi_packages_to_remove():
+    pypi_packages = [p for p in list_packages() if is_pypi(p)]
+    sorted(pypi_packages, key=lambda e: e["created_at"])
+    pypi_packages.reverse()
+    if len(pypi_packages) <= 1:
+        return []
+    return pypi_packages[1:]
+
+def push_wheel(file_path: str, try_count: int = 3):
+    if try_count <= 0:
+        sys.exit(1)
+
+    url = "https://gitlab.com/api/v4/projects/{project_id}/packages/pypi"
+    url = url.format(project_id=GITLAB_PROJECT_ID)
+
+    settings: TwineSettings = TwineSettings(
+        username="romainthomas",
+        password=LIEF_GITLAB_TOKEN,
+        repository_url=url,
+        verbose=0
+    )
+    try:
+        twine_upload.upload(settings, [file_path])
+    except requests.exceptions.HTTPError as e:
+        logger.error("%s", e)
+        if e.response.status_code == 400:
+            logger.error("-> %s", e.response.text)
+            json_info = e.response.json()
+            if json_info.get("message", "") == "Validation failed: File name has already been taken":
+                return
+        return push_wheel(file_path, try_count - 1)
+    logger.info("%s uploaded!", file_path)
+
 
 def process_pkg_files(files):
     targets     = []
@@ -302,7 +378,7 @@ def process_packages():
     keeps   = []
     removes = []
 
-    for pkg in packages:
+    for pkg in filter(is_generic, packages):
         name         = pkg["name"]
         version      = pkg["version"]
         package_type = pkg["package_type"]
@@ -359,13 +435,10 @@ def generate_index(files):
 
 for file in DIST_DIR.glob("*.whl"):
     logger.debug("[WHEEL] Uploading '%s'", file.as_posix())
-    push_file(file.as_posix(), gitlab_packages_name, gitlab_packages_version)
-
-for file in BUILD_DIR.glob("*.zip"):
-    logger.debug("[SDK  ] Uploading '%s'", file.as_posix())
-    push_file(file.as_posix(), gitlab_packages_name, gitlab_packages_version)
+    push_wheel(file.as_posix())
+    #push_file(file.as_posix(), gitlab_packages_name, gitlab_packages_version)
 
-for file in BUILD_DIR.glob("*.tar.gz"):
+for file in itertools.chain(BUILD_DIR.glob("*.zip"), BUILD_DIR.glob("*.tar.gz")):
     logger.debug("[SDK  ] Uploading '%s'", file.as_posix())
     push_file(file.as_posix(), gitlab_packages_name, gitlab_packages_version)
 
@@ -386,6 +459,9 @@ def generate_index(files):
     fdate = file["created_at"]
     logger.info("  [KEEP  ]: %s (%s)", fname, fdate)
 
+for pkg in pypi_packages_to_remove():
+    delete_package(pkg)
+
 #####################
 # Clone package repo
 #####################

diff --git a/.github/workflows/android.yml b/.github/workflows/android.yml
@@ -53,6 +53,7 @@ jobs:
           LIEF_AUTOMATIC_BUILDS_KEY: ${{ secrets.LIEF_AUTOMATIC_BUILDS_KEY }}
           LIEF_AUTOMATIC_BUILDS_IV: ${{ secrets.LIEF_AUTOMATIC_BUILDS_IV }}
           LIEF_GITLAB_TOKEN: ${{ secrets.LIEF_GITLAB_TOKEN }}
+          GITLAB_PROJECT_ID: ${{ secrets.GITLAB_PROJECT_ID }}
         shell: bash
         run: |
           docker run \
@@ -64,6 +65,7 @@ jobs:
             -e LIEF_AUTOMATIC_BUILDS_KEY=$LIEF_AUTOMATIC_BUILDS_KEY \
             -e LIEF_AUTOMATIC_BUILDS_IV=$LIEF_AUTOMATIC_BUILDS_IV \
             -e LIEF_GITLAB_TOKEN=$LIEF_GITLAB_TOKEN \
+            -e GITLAB_PROJECT_ID=$GITLAB_PROJECT_ID \
             --rm \
             liefproject/deploy python3 /src/.github/deploy.py
 

diff --git a/.github/workflows/ios.yml b/.github/workflows/ios.yml
@@ -52,9 +52,10 @@ jobs:
           LIEF_AUTOMATIC_BUILDS_KEY: ${{ secrets.LIEF_AUTOMATIC_BUILDS_KEY }}
           LIEF_AUTOMATIC_BUILDS_IV: ${{ secrets.LIEF_AUTOMATIC_BUILDS_IV }}
           LIEF_GITLAB_TOKEN: ${{ secrets.LIEF_GITLAB_TOKEN }}
+          GITLAB_PROJECT_ID: ${{ secrets.GITLAB_PROJECT_ID }}
         shell: bash
         run: |
-          python -m pip install --upgrade requests mako
+          python -m pip install --upgrade requests mako twine
           python .github/deploy.py
 
 

diff --git a/.github/workflows/linux-aarch64.yml b/.github/workflows/linux-aarch64.yml
@@ -68,6 +68,7 @@ jobs:
           LIEF_AUTOMATIC_BUILDS_KEY: ${{ secrets.LIEF_AUTOMATIC_BUILDS_KEY }}
           LIEF_AUTOMATIC_BUILDS_IV: ${{ secrets.LIEF_AUTOMATIC_BUILDS_IV }}
           LIEF_GITLAB_TOKEN: ${{ secrets.LIEF_GITLAB_TOKEN }}
+          GITLAB_PROJECT_ID: ${{ secrets.GITLAB_PROJECT_ID }}
         shell: bash
         run: |
           docker run \
@@ -79,6 +80,7 @@ jobs:
             -e LIEF_AUTOMATIC_BUILDS_KEY=$LIEF_AUTOMATIC_BUILDS_KEY \
             -e LIEF_AUTOMATIC_BUILDS_IV=$LIEF_AUTOMATIC_BUILDS_IV \
             -e LIEF_GITLAB_TOKEN=$LIEF_GITLAB_TOKEN \
+            -e GITLAB_PROJECT_ID=$GITLAB_PROJECT_ID \
             --rm \
             liefproject/deploy python3 /src/.github/deploy.py
 

diff --git a/.github/workflows/linux-x86-64.yml b/.github/workflows/linux-x86-64.yml
@@ -108,6 +108,7 @@ jobs:
           LIEF_AUTOMATIC_BUILDS_KEY: ${{ secrets.LIEF_AUTOMATIC_BUILDS_KEY }}
           LIEF_AUTOMATIC_BUILDS_IV: ${{ secrets.LIEF_AUTOMATIC_BUILDS_IV }}
           LIEF_GITLAB_TOKEN: ${{ secrets.LIEF_GITLAB_TOKEN }}
+          GITLAB_PROJECT_ID: ${{ secrets.GITLAB_PROJECT_ID }}
         shell: bash
         run: |
           docker run \
@@ -119,6 +120,7 @@ jobs:
             -e LIEF_AUTOMATIC_BUILDS_KEY=$LIEF_AUTOMATIC_BUILDS_KEY \
             -e LIEF_AUTOMATIC_BUILDS_IV=$LIEF_AUTOMATIC_BUILDS_IV \
             -e LIEF_GITLAB_TOKEN=$LIEF_GITLAB_TOKEN \
+            -e GITLAB_PROJECT_ID=$GITLAB_PROJECT_ID \
             --rm \
             liefproject/deploy python3 /src/.github/deploy.py
       - name: Trigger documentation

diff --git a/.github/workflows/osx.yml b/.github/workflows/osx.yml
@@ -86,9 +86,10 @@ jobs:
           LIEF_AUTOMATIC_BUILDS_KEY: ${{ secrets.LIEF_AUTOMATIC_BUILDS_KEY }}
           LIEF_AUTOMATIC_BUILDS_IV: ${{ secrets.LIEF_AUTOMATIC_BUILDS_IV }}
           LIEF_GITLAB_TOKEN: ${{ secrets.LIEF_GITLAB_TOKEN }}
+          GITLAB_PROJECT_ID: ${{ secrets.GITLAB_PROJECT_ID }}
         shell: bash
         run: |
-          python -m pip install --upgrade requests mako
+          python -m pip install --upgrade requests mako twine
           python .github/deploy.py
 
 

diff --git a/.github/workflows/windows-x64-all.yml b/.github/workflows/windows-x64-all.yml
@@ -107,11 +107,12 @@ jobs:
         LIEF_AUTOMATIC_BUILDS_KEY: ${{ secrets.LIEF_AUTOMATIC_BUILDS_KEY }}
         LIEF_AUTOMATIC_BUILDS_IV: ${{ secrets.LIEF_AUTOMATIC_BUILDS_IV }}
         LIEF_GITLAB_TOKEN: ${{ secrets.LIEF_GITLAB_TOKEN }}
+        GITLAB_PROJECT_ID: ${{ secrets.GITLAB_PROJECT_ID }}
         GITHUB_ACTIONS: "true"
         GITHUB_WORKSPACE: ${{ env.GITHUB_WORKSPACE }}
         GITHUB_REF: ${{ env.GITHUB_REF }}
         GITHUB_REPOSITORY: ${{ env.GITHUB_REPOSITORY }}
-        WSLENV: LIEF_AUTOMATIC_BUILDS_KEY:LIEF_AUTOMATIC_BUILDS_IV:LIEF_GITLAB_TOKEN:GITHUB_ACTIONS:GITHUB_WORKSPACE:GITHUB_REF:GITHUB_REPOSITORY
+        WSLENV: LIEF_AUTOMATIC_BUILDS_KEY:LIEF_AUTOMATIC_BUILDS_IV:LIEF_GITLAB_TOKEN:GITLAB_PROJECT_ID:GITHUB_ACTIONS:GITHUB_WORKSPACE:GITHUB_REF:GITHUB_REPOSITORY
       run: |
-        python3 -m pip install --no-cache-dir --upgrade pip setuptools requests mako wheel
+        python3 -m pip install --no-cache-dir --upgrade pip setuptools requests mako wheel twine
         python3 .github/deploy.py
diff --git a/.github/workflows/windows-x86-all.yml b/.github/workflows/windows-x86-all.yml
@@ -107,11 +107,12 @@ jobs:
         LIEF_AUTOMATIC_BUILDS_KEY: ${{ secrets.LIEF_AUTOMATIC_BUILDS_KEY }}
         LIEF_AUTOMATIC_BUILDS_IV: ${{ secrets.LIEF_AUTOMATIC_BUILDS_IV }}
         LIEF_GITLAB_TOKEN: ${{ secrets.LIEF_GITLAB_TOKEN }}
+        GITLAB_PROJECT_ID: ${{ secrets.GITLAB_PROJECT_ID }}
         GITHUB_ACTIONS: "true"
         GITHUB_WORKSPACE: ${{ env.GITHUB_WORKSPACE }}
         GITHUB_REF: ${{ env.GITHUB_REF }}
         GITHUB_REPOSITORY: ${{ env.GITHUB_REPOSITORY }}
-        WSLENV: LIEF_AUTOMATIC_BUILDS_KEY:LIEF_AUTOMATIC_BUILDS_IV:LIEF_GITLAB_TOKEN:GITHUB_ACTIONS:GITHUB_WORKSPACE:GITHUB_REF:GITHUB_REPOSITORY
+        WSLENV: LIEF_AUTOMATIC_BUILDS_KEY:LIEF_AUTOMATIC_BUILDS_IV:LIEF_GITLAB_TOKEN:GITLAB_PROJECT_ID:GITHUB_ACTIONS:GITHUB_WORKSPACE:GITHUB_REF:GITHUB_REPOSITORY
       run: |
-        python3 -m pip install --no-cache-dir --upgrade pip setuptools requests mako wheel
+        python3 -m pip install --no-cache-dir --upgrade pip setuptools requests mako wheel twine
         python3 .github/deploy.py
diff --git a/setup.py b/setup.py
@@ -411,7 +411,7 @@ def distutils_get_config_vars(*args):
 # From setuptools-git-version
 command       = 'git describe --tags --long --dirty'
 is_tagged_cmd = 'git tag --list --points-at=HEAD'
-fmt_dev       = '{tag}.dev0'
+fmt_dev       = '{tag}+{gitsha}'
 fmt_tagged    = '{tag}'
 
 def format_version(version: str, fmt: str = fmt_dev, is_dev: bool = False):