Skip to content

Commit

Permalink
Use Gitlab PyPi package
Browse files Browse the repository at this point in the history
  • Loading branch information
romainthomas committed Mar 28, 2022
1 parent 2e9ed7c commit e72a173
Show file tree
Hide file tree
Showing 9 changed files with 98 additions and 15 deletions.
89 changes: 81 additions & 8 deletions .github/deploy.py
Original file line number Diff line number Diff line change
Expand Up @@ -8,10 +8,15 @@
import shutil
import json
import requests
import itertools
from collections import defaultdict
from datetime import datetime
from mako.template import Template
from enum import Enum, auto
import twine

from twine.settings import Settings as TwineSettings
from twine.commands import upload as twine_upload

class CI(Enum):
UNKNOWN = auto()
Expand Down Expand Up @@ -125,7 +130,7 @@ def get_tag(ci):
CI_PRETTY_NAME = pretty_ci_name(CURRENT_CI)
logger.info("CI: %s", CI_PRETTY_NAME)

ALLOWED_BRANCHES = {"master", "deploy", "devel"}
ALLOWED_BRANCHES = {"master", "deploy", "devel", "feat/gitlab-packages"}
BRANCH_NAME = get_branch(CURRENT_CI)
TAG_NAME = get_tag(CURRENT_CI)
IS_TAGGED = TAG_NAME is not None and len(TAG_NAME) > 0
Expand Down Expand Up @@ -178,7 +183,11 @@ def get_tag(ci):
BUILD_DIR = REPODIR / "build"

# Gitlab
GITLAB_PROJECT_ID = 34840840 # https://gitlab.com/lief-project/packages
GITLAB_PROJECT_ID = int(os.getenv("GITLAB_PROJECT_ID", 0)) # https://gitlab.com/lief-project/packages

if GITLAB_PROJECT_ID == 0:
logger.error("GITLAB_PROJECT_ID is not set!")
sys.exit(1)

logger.debug("Working directory: %s", CI_CWD)

Expand All @@ -205,6 +214,27 @@ def get_tag(ci):
</html>
"""

class NotFound(Exception):
pass

def delete_package(info):
logger.info("Deleting %s [%s] (%s)", info["package_type"], info["name"], info["version"])
url = "https://gitlab.com/api/v4/projects/{project_id}/packages/{id}"
headers = {
"PRIVATE-TOKEN": LIEF_GITLAB_TOKEN
}

url = url.format(
project_id=GITLAB_PROJECT_ID,
id=info["id"],
)
r = requests.delete(url, headers=headers)

if r.status_code != 204:
logger.error("Error while deleting the package (%s)", r.text)
raise NotFound


def delete_file(file_info):
logger.info("Deleting %s [%s]", file_info["file_name"], file_info["created_at"])
url = "https://gitlab.com/api/v4/projects/{project_id}/packages/{pkd_id}/package_files/{file_id}"
Expand Down Expand Up @@ -275,6 +305,49 @@ def push_file(file_path: str, pkg_name: str, version: str):

return r.json()

def is_pypi(info) -> bool:
return info["package_type"] == "pypi"

def delete_pypi():
for pkg in filter(is_pypi, list_packages()):
try:
delete_package(pkg)
except NotFound:
continue

def pypi_packages_to_remove():
pypi_packages = [p for p in list_packages() if is_pypi(p)]
sorted(pypi_packages, key=lambda e: e["created_at"])
pypi_packages.reverse()
if len(pypi_packages) <= 1:
return []
return pypi_packages[1:]

def push_wheel(file_path: str, try_count: int = 3):
if try_count <= 0:
sys.exit(1)

url = "https://gitlab.com/api/v4/projects/{project_id}/packages/pypi"
url = url.format(project_id=GITLAB_PROJECT_ID)

settings: TwineSettings = TwineSettings(
username="romainthomas",
password=LIEF_GITLAB_TOKEN,
repository_url=url,
verbose=0
)
try:
twine_upload.upload(settings, [file_path])
except requests.exceptions.HTTPError as e:
logger.error("%s", e)
if e.response.status_code == 400:
logger.error("-> %s", e.response.text)
json_info = e.response.json()
if json_info.get("message", "") == "Validation failed: File name has already been taken":
return
return push_wheel(file_path, try_count - 1)
logger.info("%s uploaded!", file_path)


def process_pkg_files(files):
targets = []
Expand Down Expand Up @@ -359,13 +432,10 @@ def generate_index(files):

for file in DIST_DIR.glob("*.whl"):
logger.debug("[WHEEL] Uploading '%s'", file.as_posix())
push_file(file.as_posix(), gitlab_packages_name, gitlab_packages_version)

for file in BUILD_DIR.glob("*.zip"):
logger.debug("[SDK ] Uploading '%s'", file.as_posix())
push_file(file.as_posix(), gitlab_packages_name, gitlab_packages_version)
push_wheel(file.as_posix())
#push_file(file.as_posix(), gitlab_packages_name, gitlab_packages_version)

for file in BUILD_DIR.glob("*.tar.gz"):
for file in itertools.chain(BUILD_DIR.glob("*.zip"), BUILD_DIR.glob("*.tar.gz")):
logger.debug("[SDK ] Uploading '%s'", file.as_posix())
push_file(file.as_posix(), gitlab_packages_name, gitlab_packages_version)

Expand All @@ -386,6 +456,9 @@ def generate_index(files):
fdate = file["created_at"]
logger.info(" [KEEP ]: %s (%s)", fname, fdate)

for pkg in pypi_packages_to_remove():
delete_package(pkg)

#####################
# Clone package repo
#####################
Expand Down
2 changes: 2 additions & 0 deletions .github/workflows/android.yml
Original file line number Diff line number Diff line change
Expand Up @@ -53,6 +53,7 @@ jobs:
LIEF_AUTOMATIC_BUILDS_KEY: ${{ secrets.LIEF_AUTOMATIC_BUILDS_KEY }}
LIEF_AUTOMATIC_BUILDS_IV: ${{ secrets.LIEF_AUTOMATIC_BUILDS_IV }}
LIEF_GITLAB_TOKEN: ${{ secrets.LIEF_GITLAB_TOKEN }}
GITLAB_PROJECT_ID: ${{ secrets.GITLAB_PROJECT_ID }}
shell: bash
run: |
docker run \
Expand All @@ -64,6 +65,7 @@ jobs:
-e LIEF_AUTOMATIC_BUILDS_KEY=$LIEF_AUTOMATIC_BUILDS_KEY \
-e LIEF_AUTOMATIC_BUILDS_IV=$LIEF_AUTOMATIC_BUILDS_IV \
-e LIEF_GITLAB_TOKEN=$LIEF_GITLAB_TOKEN \
-e GITLAB_PROJECT_ID=$GITLAB_PROJECT_ID \
--rm \
liefproject/deploy python3 /src/.github/deploy.py
Expand Down
3 changes: 2 additions & 1 deletion .github/workflows/ios.yml
Original file line number Diff line number Diff line change
Expand Up @@ -52,9 +52,10 @@ jobs:
LIEF_AUTOMATIC_BUILDS_KEY: ${{ secrets.LIEF_AUTOMATIC_BUILDS_KEY }}
LIEF_AUTOMATIC_BUILDS_IV: ${{ secrets.LIEF_AUTOMATIC_BUILDS_IV }}
LIEF_GITLAB_TOKEN: ${{ secrets.LIEF_GITLAB_TOKEN }}
GITLAB_PROJECT_ID: ${{ secrets.GITLAB_PROJECT_ID }}
shell: bash
run: |
python -m pip install --upgrade requests mako
python -m pip install --upgrade requests mako twine
python .github/deploy.py
Expand Down
2 changes: 2 additions & 0 deletions .github/workflows/linux-aarch64.yml
Original file line number Diff line number Diff line change
Expand Up @@ -68,6 +68,7 @@ jobs:
LIEF_AUTOMATIC_BUILDS_KEY: ${{ secrets.LIEF_AUTOMATIC_BUILDS_KEY }}
LIEF_AUTOMATIC_BUILDS_IV: ${{ secrets.LIEF_AUTOMATIC_BUILDS_IV }}
LIEF_GITLAB_TOKEN: ${{ secrets.LIEF_GITLAB_TOKEN }}
GITLAB_PROJECT_ID: ${{ secrets.GITLAB_PROJECT_ID }}
shell: bash
run: |
docker run \
Expand All @@ -79,6 +80,7 @@ jobs:
-e LIEF_AUTOMATIC_BUILDS_KEY=$LIEF_AUTOMATIC_BUILDS_KEY \
-e LIEF_AUTOMATIC_BUILDS_IV=$LIEF_AUTOMATIC_BUILDS_IV \
-e LIEF_GITLAB_TOKEN=$LIEF_GITLAB_TOKEN \
-e GITLAB_PROJECT_ID=$GITLAB_PROJECT_ID \
--rm \
liefproject/deploy python3 /src/.github/deploy.py
Expand Down
2 changes: 2 additions & 0 deletions .github/workflows/linux-x86-64.yml
Original file line number Diff line number Diff line change
Expand Up @@ -108,6 +108,7 @@ jobs:
LIEF_AUTOMATIC_BUILDS_KEY: ${{ secrets.LIEF_AUTOMATIC_BUILDS_KEY }}
LIEF_AUTOMATIC_BUILDS_IV: ${{ secrets.LIEF_AUTOMATIC_BUILDS_IV }}
LIEF_GITLAB_TOKEN: ${{ secrets.LIEF_GITLAB_TOKEN }}
GITLAB_PROJECT_ID: ${{ secrets.GITLAB_PROJECT_ID }}
shell: bash
run: |
docker run \
Expand All @@ -119,6 +120,7 @@ jobs:
-e LIEF_AUTOMATIC_BUILDS_KEY=$LIEF_AUTOMATIC_BUILDS_KEY \
-e LIEF_AUTOMATIC_BUILDS_IV=$LIEF_AUTOMATIC_BUILDS_IV \
-e LIEF_GITLAB_TOKEN=$LIEF_GITLAB_TOKEN \
-e GITLAB_PROJECT_ID=$GITLAB_PROJECT_ID \
--rm \
liefproject/deploy python3 /src/.github/deploy.py
- name: Trigger documentation
Expand Down
3 changes: 2 additions & 1 deletion .github/workflows/osx.yml
Original file line number Diff line number Diff line change
Expand Up @@ -86,9 +86,10 @@ jobs:
LIEF_AUTOMATIC_BUILDS_KEY: ${{ secrets.LIEF_AUTOMATIC_BUILDS_KEY }}
LIEF_AUTOMATIC_BUILDS_IV: ${{ secrets.LIEF_AUTOMATIC_BUILDS_IV }}
LIEF_GITLAB_TOKEN: ${{ secrets.LIEF_GITLAB_TOKEN }}
GITLAB_PROJECT_ID: ${{ secrets.GITLAB_PROJECT_ID }}
shell: bash
run: |
python -m pip install --upgrade requests mako
python -m pip install --upgrade requests mako twine
python .github/deploy.py
Expand Down
5 changes: 3 additions & 2 deletions .github/workflows/windows-x64-all.yml
Original file line number Diff line number Diff line change
Expand Up @@ -107,11 +107,12 @@ jobs:
LIEF_AUTOMATIC_BUILDS_KEY: ${{ secrets.LIEF_AUTOMATIC_BUILDS_KEY }}
LIEF_AUTOMATIC_BUILDS_IV: ${{ secrets.LIEF_AUTOMATIC_BUILDS_IV }}
LIEF_GITLAB_TOKEN: ${{ secrets.LIEF_GITLAB_TOKEN }}
GITLAB_PROJECT_ID: ${{ secrets.GITLAB_PROJECT_ID }}
GITHUB_ACTIONS: "true"
GITHUB_WORKSPACE: ${{ env.GITHUB_WORKSPACE }}
GITHUB_REF: ${{ env.GITHUB_REF }}
GITHUB_REPOSITORY: ${{ env.GITHUB_REPOSITORY }}
WSLENV: LIEF_AUTOMATIC_BUILDS_KEY:LIEF_AUTOMATIC_BUILDS_IV:LIEF_GITLAB_TOKEN:GITHUB_ACTIONS:GITHUB_WORKSPACE:GITHUB_REF:GITHUB_REPOSITORY
WSLENV: LIEF_AUTOMATIC_BUILDS_KEY:LIEF_AUTOMATIC_BUILDS_IV:LIEF_GITLAB_TOKEN:GITLAB_PROJECT_ID:GITHUB_ACTIONS:GITHUB_WORKSPACE:GITHUB_REF:GITHUB_REPOSITORY
run: |
python3 -m pip install --no-cache-dir --upgrade pip setuptools requests mako wheel
python3 -m pip install --no-cache-dir --upgrade pip setuptools requests mako wheel twine
python3 .github/deploy.py
5 changes: 3 additions & 2 deletions .github/workflows/windows-x86-all.yml
Original file line number Diff line number Diff line change
Expand Up @@ -107,11 +107,12 @@ jobs:
LIEF_AUTOMATIC_BUILDS_KEY: ${{ secrets.LIEF_AUTOMATIC_BUILDS_KEY }}
LIEF_AUTOMATIC_BUILDS_IV: ${{ secrets.LIEF_AUTOMATIC_BUILDS_IV }}
LIEF_GITLAB_TOKEN: ${{ secrets.LIEF_GITLAB_TOKEN }}
GITLAB_PROJECT_ID: ${{ secrets.GITLAB_PROJECT_ID }}
GITHUB_ACTIONS: "true"
GITHUB_WORKSPACE: ${{ env.GITHUB_WORKSPACE }}
GITHUB_REF: ${{ env.GITHUB_REF }}
GITHUB_REPOSITORY: ${{ env.GITHUB_REPOSITORY }}
WSLENV: LIEF_AUTOMATIC_BUILDS_KEY:LIEF_AUTOMATIC_BUILDS_IV:LIEF_GITLAB_TOKEN:GITHUB_ACTIONS:GITHUB_WORKSPACE:GITHUB_REF:GITHUB_REPOSITORY
WSLENV: LIEF_AUTOMATIC_BUILDS_KEY:LIEF_AUTOMATIC_BUILDS_IV:LIEF_GITLAB_TOKEN:GITLAB_PROJECT_ID:GITHUB_ACTIONS:GITHUB_WORKSPACE:GITHUB_REF:GITHUB_REPOSITORY
run: |
python3 -m pip install --no-cache-dir --upgrade pip setuptools requests mako wheel
python3 -m pip install --no-cache-dir --upgrade pip setuptools requests mako wheel twine
python3 .github/deploy.py
2 changes: 1 addition & 1 deletion setup.py
Original file line number Diff line number Diff line change
Expand Up @@ -411,7 +411,7 @@ def distutils_get_config_vars(*args):
# From setuptools-git-version
command = 'git describe --tags --long --dirty'
is_tagged_cmd = 'git tag --list --points-at=HEAD'
fmt_dev = '{tag}.dev0'
fmt_dev = '{tag}+{gitsha}'
fmt_tagged = '{tag}'

def format_version(version: str, fmt: str = fmt_dev, is_dev: bool = False):
Expand Down

0 comments on commit e72a173

Please sign in to comment.