diff --git a/docker/worker/.gitignore b/docker/worker/.gitignore index 2af7e0e1267..0b26b57df2c 100644 --- a/docker/worker/.gitignore +++ b/docker/worker/.gitignore @@ -1,2 +1,3 @@ utils/ -lib/ \ No newline at end of file +lib/ +testdata/tmp \ No newline at end of file diff --git a/docker/worker/testdata/CVE-2022-27449.json b/docker/worker/testdata/CVE-2022-27449.json new file mode 100644 index 00000000000..aefab1c556e --- /dev/null +++ b/docker/worker/testdata/CVE-2022-27449.json @@ -0,0 +1,127 @@ +{ + "id": "CVE-2022-27449", + "details": "MariaDB Server v10.9 and below was discovered to contain a segmentation fault via the component sql/item_func.cc:148.", + "affected": [ + { + "package": { + "name": "mariadb", + "ecosystem": "Alpine:v3.12", + "purl": "pkg:alpine/mariadb" + }, + "ranges": [ + { + "type": "ECOSYSTEM", + "events": [ + { + "introduced": "0" + }, + { + "fixed": "10.4.25-r0" + } + ] + } + ], + "versions": null + }, + { + "package": { + "name": "mariadb", + "ecosystem": "Alpine:v3.13", + "purl": "pkg:alpine/mariadb" + }, + "ranges": [ + { + "type": "ECOSYSTEM", + "events": [ + { + "introduced": "0" + }, + { + "fixed": "10.5.16-r0" + } + ] + } + ], + "versions": null + }, + { + "package": { + "name": "mariadb", + "ecosystem": "Alpine:v3.14", + "purl": "pkg:alpine/mariadb" + }, + "ranges": [ + { + "type": "ECOSYSTEM", + "events": [ + { + "introduced": "0" + }, + { + "fixed": "10.5.16-r0" + } + ] + } + ], + "versions": null + }, + { + "package": { + "name": "mariadb", + "ecosystem": "Alpine:v3.15", + "purl": "pkg:alpine/mariadb" + }, + "ranges": [ + { + "type": "ECOSYSTEM", + "events": [ + { + "introduced": "0" + }, + { + "fixed": "10.6.8-r0" + } + ] + } + ], + "versions": null + }, + { + "package": { + "name": "mariadb", + "ecosystem": "Alpine:v3.16", + "purl": "pkg:alpine/mariadb" + }, + "ranges": [ + { + "type": "ECOSYSTEM", + "events": [ + { + "introduced": "0" + }, + { + "fixed": "10.6.8-r0" + } + ] + } + ], + "versions": null + } + ], + "references": [ + { + "type": "WEB", + "url": "https://jira.mariadb.org/browse/MDEV-28089" + }, + { + "type": "ADVISORY", + "url": "https://security.netapp.com/advisory/ntap-20220526-0006/" + }, + { + "type": "WEB", + "url": "https://lists.debian.org/debian-lts-announce/2022/09/msg00023.html" + } + ], + "modified": "2022-10-07T18:59:00Z", + "published": "2022-04-14T13:15:00Z" +} diff --git a/docker/worker/testdata/UpdateTest_diff_alpine.txt b/docker/worker/testdata/UpdateTest_diff_alpine.txt new file mode 100644 index 00000000000..bb24b120c08 --- /dev/null +++ b/docker/worker/testdata/UpdateTest_diff_alpine.txt @@ -0,0 +1,333 @@ +('diff --git a/CVE-2022-27449.json b/CVE-2022-27449.json\n' + 'index aefab1c..8f8329f 100644\n' + '--- a/CVE-2022-27449.json\n' + '+++ b/CVE-2022-27449.json\n' + '@@ -1,6 +1,22 @@\n' + ' {\n' + ' "id": "CVE-2022-27449",\n' + ' "details": "MariaDB Server v10.9 and below was discovered to contain a ' + 'segmentation fault via the component sql/item_func.cc:148.",\n' + '+ "modified": "2021-01-01T00:00:00Z",\n' + '+ "published": "2022-04-14T13:15:00Z",\n' + '+ "references": [\n' + '+ {\n' + '+ "type": "WEB",\n' + '+ "url": "https://jira.mariadb.org/browse/MDEV-28089"\n' + '+ },\n' + '+ {\n' + '+ "type": "ADVISORY",\n' + '+ "url": "https://security.netapp.com/advisory/ntap-20220526-0006/"\n' + '+ },\n' + '+ {\n' + '+ "type": "WEB",\n' + '+ "url": ' + '"https://lists.debian.org/debian-lts-announce/2022/09/msg00023.html"\n' + '+ }\n' + '+ ],\n' + ' "affected": [\n' + ' {\n' + ' "package": {\n' + '@@ -21,7 +37,53 @@\n' + ' ]\n' + ' }\n' + ' ],\n' + '- "versions": null\n' + '+ "versions": [\n' + '+ "10.0.21",\n' + '+ "10.1.11",\n' + '+ "10.1.12",\n' + '+ "10.1.13",\n' + '+ "10.1.14",\n' + '+ "10.1.16",\n' + '+ "10.1.17",\n' + '+ "10.1.18",\n' + '+ "10.1.19",\n' + '+ "10.1.20",\n' + '+ "10.1.21",\n' + '+ "10.1.22",\n' + '+ "10.1.24",\n' + '+ "10.1.26",\n' + '+ "10.1.28",\n' + '+ "10.1.31",\n' + '+ "10.1.8",\n' + '+ "10.1.9",\n' + '+ "10.2.13",\n' + '+ "10.2.14",\n' + '+ "10.2.15",\n' + '+ "10.3.10",\n' + '+ "10.3.11",\n' + '+ "10.3.12",\n' + '+ "10.3.13",\n' + '+ "10.3.15",\n' + '+ "10.3.16",\n' + '+ "10.3.9",\n' + '+ "10.4.10",\n' + '+ "10.4.11",\n' + '+ "10.4.12",\n' + '+ "10.4.13",\n' + '+ "10.4.15",\n' + '+ "10.4.17",\n' + '+ "10.4.18",\n' + '+ "10.4.19",\n' + '+ "10.4.21",\n' + '+ "10.4.22",\n' + '+ "10.4.24",\n' + '+ "10.4.6",\n' + '+ "10.4.7",\n' + '+ "10.4.8",\n' + '+ "5.5.41",\n' + '+ "5.5.42",\n' + '+ "5.5.43"\n' + '+ ]\n' + ' },\n' + ' {\n' + ' "package": {\n' + '@@ -42,7 +104,56 @@\n' + ' ]\n' + ' }\n' + ' ],\n' + '- "versions": null\n' + '+ "versions": [\n' + '+ "10.0.21",\n' + '+ "10.1.11",\n' + '+ "10.1.12",\n' + '+ "10.1.13",\n' + '+ "10.1.14",\n' + '+ "10.1.16",\n' + '+ "10.1.17",\n' + '+ "10.1.18",\n' + '+ "10.1.19",\n' + '+ "10.1.20",\n' + '+ "10.1.21",\n' + '+ "10.1.22",\n' + '+ "10.1.24",\n' + '+ "10.1.26",\n' + '+ "10.1.28",\n' + '+ "10.1.31",\n' + '+ "10.1.8",\n' + '+ "10.1.9",\n' + '+ "10.2.13",\n' + '+ "10.2.14",\n' + '+ "10.2.15",\n' + '+ "10.3.10",\n' + '+ "10.3.11",\n' + '+ "10.3.12",\n' + '+ "10.3.13",\n' + '+ "10.3.15",\n' + '+ "10.3.16",\n' + '+ "10.3.9",\n' + '+ "10.4.10",\n' + '+ "10.4.11",\n' + '+ "10.4.12",\n' + '+ "10.4.13",\n' + '+ "10.4.14",\n' + '+ "10.4.6",\n' + '+ "10.4.7",\n' + '+ "10.4.8",\n' + '+ "10.5.10",\n' + '+ "10.5.11",\n' + '+ "10.5.12",\n' + '+ "10.5.13",\n' + '+ "10.5.15",\n' + '+ "10.5.5",\n' + '+ "10.5.6",\n' + '+ "10.5.8",\n' + '+ "10.5.9",\n' + '+ "5.5.41",\n' + '+ "5.5.42",\n' + '+ "5.5.43"\n' + '+ ]\n' + ' },\n' + ' {\n' + ' "package": {\n' + '@@ -63,7 +174,55 @@\n' + ' ]\n' + ' }\n' + ' ],\n' + '- "versions": null\n' + '+ "versions": [\n' + '+ "10.0.21",\n' + '+ "10.1.11",\n' + '+ "10.1.12",\n' + '+ "10.1.13",\n' + '+ "10.1.14",\n' + '+ "10.1.16",\n' + '+ "10.1.17",\n' + '+ "10.1.18",\n' + '+ "10.1.19",\n' + '+ "10.1.20",\n' + '+ "10.1.21",\n' + '+ "10.1.22",\n' + '+ "10.1.24",\n' + '+ "10.1.26",\n' + '+ "10.1.28",\n' + '+ "10.1.31",\n' + '+ "10.1.8",\n' + '+ "10.1.9",\n' + '+ "10.2.13",\n' + '+ "10.2.14",\n' + '+ "10.2.15",\n' + '+ "10.3.10",\n' + '+ "10.3.11",\n' + '+ "10.3.12",\n' + '+ "10.3.13",\n' + '+ "10.3.15",\n' + '+ "10.3.16",\n' + '+ "10.3.9",\n' + '+ "10.4.10",\n' + '+ "10.4.11",\n' + '+ "10.4.12",\n' + '+ "10.4.13",\n' + '+ "10.4.14",\n' + '+ "10.4.6",\n' + '+ "10.4.7",\n' + '+ "10.4.8",\n' + '+ "10.5.11",\n' + '+ "10.5.12",\n' + '+ "10.5.13",\n' + '+ "10.5.15",\n' + '+ "10.5.5",\n' + '+ "10.5.6",\n' + '+ "10.5.8",\n' + '+ "10.5.9",\n' + '+ "5.5.41",\n' + '+ "5.5.42",\n' + '+ "5.5.43"\n' + '+ ]\n' + ' },\n' + ' {\n' + ' "package": {\n' + '@@ -84,7 +243,55 @@\n' + ' ]\n' + ' }\n' + ' ],\n' + '- "versions": null\n' + '+ "versions": [\n' + '+ "10.0.21",\n' + '+ "10.1.11",\n' + '+ "10.1.12",\n' + '+ "10.1.13",\n' + '+ "10.1.14",\n' + '+ "10.1.16",\n' + '+ "10.1.17",\n' + '+ "10.1.18",\n' + '+ "10.1.19",\n' + '+ "10.1.20",\n' + '+ "10.1.21",\n' + '+ "10.1.22",\n' + '+ "10.1.24",\n' + '+ "10.1.26",\n' + '+ "10.1.28",\n' + '+ "10.1.31",\n' + '+ "10.1.8",\n' + '+ "10.1.9",\n' + '+ "10.2.13",\n' + '+ "10.2.14",\n' + '+ "10.2.15",\n' + '+ "10.3.10",\n' + '+ "10.3.11",\n' + '+ "10.3.12",\n' + '+ "10.3.13",\n' + '+ "10.3.15",\n' + '+ "10.3.16",\n' + '+ "10.3.9",\n' + '+ "10.4.10",\n' + '+ "10.4.11",\n' + '+ "10.4.12",\n' + '+ "10.4.13",\n' + '+ "10.4.14",\n' + '+ "10.4.6",\n' + '+ "10.4.7",\n' + '+ "10.4.8",\n' + '+ "10.5.11",\n' + '+ "10.5.5",\n' + '+ "10.5.6",\n' + '+ "10.5.8",\n' + '+ "10.5.9",\n' + '+ "10.6.3",\n' + '+ "10.6.4",\n' + '+ "10.6.7",\n' + '+ "5.5.41",\n' + '+ "5.5.42",\n' + '+ "5.5.43"\n' + '+ ]\n' + ' },\n' + ' {\n' + ' "package": {\n' + '@@ -105,23 +312,55 @@\n' + ' ]\n' + ' }\n' + ' ],\n' + '- "versions": null\n' + '+ "versions": [\n' + '+ "10.0.21",\n' + '+ "10.1.11",\n' + '+ "10.1.12",\n' + '+ "10.1.13",\n' + '+ "10.1.14",\n' + '+ "10.1.16",\n' + '+ "10.1.17",\n' + '+ "10.1.18",\n' + '+ "10.1.19",\n' + '+ "10.1.20",\n' + '+ "10.1.21",\n' + '+ "10.1.22",\n' + '+ "10.1.24",\n' + '+ "10.1.26",\n' + '+ "10.1.28",\n' + '+ "10.1.31",\n' + '+ "10.1.8",\n' + '+ "10.1.9",\n' + '+ "10.2.13",\n' + '+ "10.2.14",\n' + '+ "10.2.15",\n' + '+ "10.3.10",\n' + '+ "10.3.11",\n' + '+ "10.3.12",\n' + '+ "10.3.13",\n' + '+ "10.3.15",\n' + '+ "10.3.16",\n' + '+ "10.3.9",\n' + '+ "10.4.10",\n' + '+ "10.4.11",\n' + '+ "10.4.12",\n' + '+ "10.4.13",\n' + '+ "10.4.14",\n' + '+ "10.4.6",\n' + '+ "10.4.7",\n' + '+ "10.4.8",\n' + '+ "10.5.11",\n' + '+ "10.5.5",\n' + '+ "10.5.6",\n' + '+ "10.5.8",\n' + '+ "10.5.9",\n' + '+ "10.6.3",\n' + '+ "10.6.4",\n' + '+ "10.6.7",\n' + '+ "5.5.41",\n' + '+ "5.5.42",\n' + '+ "5.5.43"\n' + '+ ]\n' + ' }\n' + '- ],\n' + '- "references": [\n' + '- {\n' + '- "type": "WEB",\n' + '- "url": "https://jira.mariadb.org/browse/MDEV-28089"\n' + '- },\n' + '- {\n' + '- "type": "ADVISORY",\n' + '- "url": "https://security.netapp.com/advisory/ntap-20220526-0006/"\n' + '- },\n' + '- {\n' + '- "type": "WEB",\n' + '- "url": ' + '"https://lists.debian.org/debian-lts-announce/2022/09/msg00023.html"\n' + '- }\n' + '- ],\n' + '- "modified": "2022-10-07T18:59:00Z",\n' + '- "published": "2022-04-14T13:15:00Z"\n' + '-}\n' + '+ ]\n' + '+}\n' + '\\ No newline at end of file\n') \ No newline at end of file diff --git a/docker/worker/testdata/UpdateTest_update_alpine.txt b/docker/worker/testdata/UpdateTest_update_alpine.txt new file mode 100644 index 00000000000..a36ee95998b --- /dev/null +++ b/docker/worker/testdata/UpdateTest_update_alpine.txt @@ -0,0 +1,400 @@ +{ 'affected': [], + 'affected_fuzzy': [ '10.0.21', + '10.1.11', + '10.1.12', + '10.1.13', + '10.1.14', + '10.1.16', + '10.1.17', + '10.1.18', + '10.1.19', + '10.1.20', + '10.1.21', + '10.1.22', + '10.1.24', + '10.1.26', + '10.1.28', + '10.1.31', + '10.1.8', + '10.1.9', + '10.2.13', + '10.2.14', + '10.2.15', + '10.3.10', + '10.3.11', + '10.3.12', + '10.3.13', + '10.3.15', + '10.3.16', + '10.3.9', + '10.4.10', + '10.4.11', + '10.4.12', + '10.4.13', + '10.4.14', + '10.4.15', + '10.4.17', + '10.4.18', + '10.4.19', + '10.4.21', + '10.4.22', + '10.4.24', + '10.4.6', + '10.4.7', + '10.4.8', + '10.5.10', + '10.5.11', + '10.5.12', + '10.5.13', + '10.5.15', + '10.5.5', + '10.5.6', + '10.5.8', + '10.5.9', + '10.6.3', + '10.6.4', + '10.6.7', + '5.5.41', + '5.5.42', + '5.5.43'], + 'affected_packages': [ { 'database_specific': None, + 'ecosystem_specific': None, + 'package': { 'ecosystem': 'Alpine:v3.12', + 'name': 'mariadb', + 'purl': 'pkg:alpine/mariadb'}, + 'ranges': [ { 'events': [ { 'type': 'introduced', + 'value': '0'}, + { 'type': 'fixed', + 'value': '10.4.25-r0'}], + 'repo_url': '', + 'type': 'ECOSYSTEM'}], + 'versions': [ '10.0.21', + '10.1.11', + '10.1.12', + '10.1.13', + '10.1.14', + '10.1.16', + '10.1.17', + '10.1.18', + '10.1.19', + '10.1.20', + '10.1.21', + '10.1.22', + '10.1.24', + '10.1.26', + '10.1.28', + '10.1.31', + '10.1.8', + '10.1.9', + '10.2.13', + '10.2.14', + '10.2.15', + '10.3.10', + '10.3.11', + '10.3.12', + '10.3.13', + '10.3.15', + '10.3.16', + '10.3.9', + '10.4.10', + '10.4.11', + '10.4.12', + '10.4.13', + '10.4.15', + '10.4.17', + '10.4.18', + '10.4.19', + '10.4.21', + '10.4.22', + '10.4.24', + '10.4.6', + '10.4.7', + '10.4.8', + '5.5.41', + '5.5.42', + '5.5.43']}, + { 'database_specific': None, + 'ecosystem_specific': None, + 'package': { 'ecosystem': 'Alpine:v3.13', + 'name': 'mariadb', + 'purl': 'pkg:alpine/mariadb'}, + 'ranges': [ { 'events': [ { 'type': 'introduced', + 'value': '0'}, + { 'type': 'fixed', + 'value': '10.5.16-r0'}], + 'repo_url': '', + 'type': 'ECOSYSTEM'}], + 'versions': [ '10.0.21', + '10.1.11', + '10.1.12', + '10.1.13', + '10.1.14', + '10.1.16', + '10.1.17', + '10.1.18', + '10.1.19', + '10.1.20', + '10.1.21', + '10.1.22', + '10.1.24', + '10.1.26', + '10.1.28', + '10.1.31', + '10.1.8', + '10.1.9', + '10.2.13', + '10.2.14', + '10.2.15', + '10.3.10', + '10.3.11', + '10.3.12', + '10.3.13', + '10.3.15', + '10.3.16', + '10.3.9', + '10.4.10', + '10.4.11', + '10.4.12', + '10.4.13', + '10.4.14', + '10.4.6', + '10.4.7', + '10.4.8', + '10.5.10', + '10.5.11', + '10.5.12', + '10.5.13', + '10.5.15', + '10.5.5', + '10.5.6', + '10.5.8', + '10.5.9', + '5.5.41', + '5.5.42', + '5.5.43']}, + { 'database_specific': None, + 'ecosystem_specific': None, + 'package': { 'ecosystem': 'Alpine:v3.14', + 'name': 'mariadb', + 'purl': 'pkg:alpine/mariadb'}, + 'ranges': [ { 'events': [ { 'type': 'introduced', + 'value': '0'}, + { 'type': 'fixed', + 'value': '10.5.16-r0'}], + 'repo_url': '', + 'type': 'ECOSYSTEM'}], + 'versions': [ '10.0.21', + '10.1.11', + '10.1.12', + '10.1.13', + '10.1.14', + '10.1.16', + '10.1.17', + '10.1.18', + '10.1.19', + '10.1.20', + '10.1.21', + '10.1.22', + '10.1.24', + '10.1.26', + '10.1.28', + '10.1.31', + '10.1.8', + '10.1.9', + '10.2.13', + '10.2.14', + '10.2.15', + '10.3.10', + '10.3.11', + '10.3.12', + '10.3.13', + '10.3.15', + '10.3.16', + '10.3.9', + '10.4.10', + '10.4.11', + '10.4.12', + '10.4.13', + '10.4.14', + '10.4.6', + '10.4.7', + '10.4.8', + '10.5.11', + '10.5.12', + '10.5.13', + '10.5.15', + '10.5.5', + '10.5.6', + '10.5.8', + '10.5.9', + '5.5.41', + '5.5.42', + '5.5.43']}, + { 'database_specific': None, + 'ecosystem_specific': None, + 'package': { 'ecosystem': 'Alpine:v3.15', + 'name': 'mariadb', + 'purl': 'pkg:alpine/mariadb'}, + 'ranges': [ { 'events': [ { 'type': 'introduced', + 'value': '0'}, + { 'type': 'fixed', + 'value': '10.6.8-r0'}], + 'repo_url': '', + 'type': 'ECOSYSTEM'}], + 'versions': [ '10.0.21', + '10.1.11', + '10.1.12', + '10.1.13', + '10.1.14', + '10.1.16', + '10.1.17', + '10.1.18', + '10.1.19', + '10.1.20', + '10.1.21', + '10.1.22', + '10.1.24', + '10.1.26', + '10.1.28', + '10.1.31', + '10.1.8', + '10.1.9', + '10.2.13', + '10.2.14', + '10.2.15', + '10.3.10', + '10.3.11', + '10.3.12', + '10.3.13', + '10.3.15', + '10.3.16', + '10.3.9', + '10.4.10', + '10.4.11', + '10.4.12', + '10.4.13', + '10.4.14', + '10.4.6', + '10.4.7', + '10.4.8', + '10.5.11', + '10.5.5', + '10.5.6', + '10.5.8', + '10.5.9', + '10.6.3', + '10.6.4', + '10.6.7', + '5.5.41', + '5.5.42', + '5.5.43']}, + { 'database_specific': None, + 'ecosystem_specific': None, + 'package': { 'ecosystem': 'Alpine:v3.16', + 'name': 'mariadb', + 'purl': 'pkg:alpine/mariadb'}, + 'ranges': [ { 'events': [ { 'type': 'introduced', + 'value': '0'}, + { 'type': 'fixed', + 'value': '10.6.8-r0'}], + 'repo_url': '', + 'type': 'ECOSYSTEM'}], + 'versions': [ '10.0.21', + '10.1.11', + '10.1.12', + '10.1.13', + '10.1.14', + '10.1.16', + '10.1.17', + '10.1.18', + '10.1.19', + '10.1.20', + '10.1.21', + '10.1.22', + '10.1.24', + '10.1.26', + '10.1.28', + '10.1.31', + '10.1.8', + '10.1.9', + '10.2.13', + '10.2.14', + '10.2.15', + '10.3.10', + '10.3.11', + '10.3.12', + '10.3.13', + '10.3.15', + '10.3.16', + '10.3.9', + '10.4.10', + '10.4.11', + '10.4.12', + '10.4.13', + '10.4.14', + '10.4.6', + '10.4.7', + '10.4.8', + '10.5.11', + '10.5.5', + '10.5.6', + '10.5.8', + '10.5.9', + '10.6.3', + '10.6.4', + '10.6.7', + '5.5.41', + '5.5.42', + '5.5.43']}], + 'aliases': [], + 'credits': [], + 'database_specific': None, + 'db_id': 'CVE-2022-27449', + 'details': 'MariaDB Server v10.9 and below was discovered to contain a ' + 'segmentation fault via the component sql/item_func.cc:148.', + 'ecosystem': [ 'Alpine', + 'Alpine:v3.12', + 'Alpine:v3.13', + 'Alpine:v3.14', + 'Alpine:v3.15', + 'Alpine:v3.16'], + 'fixed': '', + 'has_affected': True, + 'import_last_modified': datetime.datetime(2022, 10, 7, 18, 59), + 'is_fixed': True, + 'issue_id': None, + 'last_modified': datetime.datetime(2021, 1, 1, 0, 0), + 'project': ['mariadb'], + 'public': True, + 'purl': ['pkg:alpine/mariadb'], + 'reference_url_types': { 'https://jira.mariadb.org/browse/MDEV-28089': 'WEB', + 'https://lists.debian.org/debian-lts-announce/2022/09/msg00023.html': 'WEB', + 'https://security.netapp.com/advisory/ntap-20220526-0006/': 'ADVISORY'}, + 'regressed': '', + 'related': [], + 'search_indices': [ '12', + '13', + '14', + '15', + '16', + '2022', + '27449', + 'alpine', + 'alpine:v3.12', + 'alpine:v3.13', + 'alpine:v3.14', + 'alpine:v3.15', + 'alpine:v3.16', + 'cve', + 'cve-2022-27449', + 'mariadb', + 'v3'], + 'semver_fixed_indexes': [], + 'severities': [], + 'source': 'source', + 'source_id': 'source:CVE-2022-27449.json', + 'source_of_truth': 2, + 'status': 1, + 'summary': '', + 'timestamp': datetime.datetime(2022, 4, 14, 13, 15), + 'withdrawn': None} \ No newline at end of file diff --git a/docker/worker/worker.py b/docker/worker/worker.py index 936097056e8..171779954a5 100644 --- a/docker/worker/worker.py +++ b/docker/worker/worker.py @@ -646,6 +646,8 @@ def main(): if args.redis_host: osv.ecosystems.set_cache(RedisCache(args.redis_host, args.redis_port)) + osv.ecosystems.work_dir = args.work_dir + # Work around kernel bug: https://gvisor.dev/issue/1765 resource.setrlimit(resource.RLIMIT_MEMLOCK, (resource.RLIM_INFINITY, resource.RLIM_INFINITY)) diff --git a/docker/worker/worker_test.py b/docker/worker/worker_test.py index 61c8f3be0e1..992788326ae 100644 --- a/docker/worker/worker_test.py +++ b/docker/worker/worker_test.py @@ -607,6 +607,8 @@ def setUp(self): self.mock_publish = mock_publish.start() self.addCleanup(mock_publish.stop) + osv.ecosystems.work_dir = 'testdata/tmp/' + def tearDown(self): self.tmp_dir.cleanup() @@ -1099,6 +1101,45 @@ def test_update_debian(self): self.mock_publish.assert_not_called() + def test_update_alpine(self): + """Test updating debian.""" + self.source_repo.ignore_git = False + self.source_repo.versions_from_repo = False + self.source_repo.detect_cherrypicks = False + self.source_repo.put() + + self.mock_repo.add_file( + 'CVE-2022-27449.json', + self._load_test_data( + os.path.join(TEST_DATA_DIR, 'CVE-2022-27449.json'))) + self.mock_repo.commit('User', 'user@email') + task_runner = worker.TaskRunner(ndb_client, None, self.tmp_dir.name, None, + None) + message = mock.Mock() + message.attributes = { + 'source': 'source', + 'path': 'CVE-2022-27449.json', + 'original_sha256': _sha256('CVE-2022-27449.json'), + 'deleted': 'false', + } + task_runner._source_update(message) + + repo = pygit2.Repository(self.remote_source_repo_path) + commit = repo.head.peel() + + self.assertEqual('infra@osv.dev', commit.author.email) + self.assertEqual('OSV', commit.author.name) + self.assertEqual('Update CVE-2022-27449', commit.message) + diff = repo.diff(commit.parents[0], commit) + + self.expect_equal('diff_alpine', diff.patch) + + self.expect_dict_equal( + 'update_alpine', + ndb.Key(osv.Bug, 'source:CVE-2022-27449').get()._to_dict()) + + self.mock_publish.assert_not_called() + def test_update_android(self): """Test updating Android through bucket entries.""" self.source_repo.type = osv.SourceRepositoryType.BUCKET diff --git a/osv/ecosystems.py b/osv/ecosystems.py index d034420060e..a4c6976da53 100644 --- a/osv/ecosystems.py +++ b/osv/ecosystems.py @@ -14,8 +14,13 @@ """Ecosystem helpers.""" import bisect +import glob import json import logging +import os.path +import subprocess +import typing +from abc import ABC, abstractmethod import packaging.version import urllib.parse @@ -23,8 +28,10 @@ from .third_party.univers.debian import Version as DebianVersion from .third_party.univers.gem import GemVersion +from .third_party.univers.alpine import AlpineLinuxVersion from . import debian_version_cache +from . import repos from . import maven from . import nuget from . import packagist_version @@ -39,51 +46,18 @@ TIMEOUT = 30 # Timeout for HTTP(S) requests use_deps_dev = False deps_dev_api_key = '' +# Used for checking out git repositories +# Intended to be set in worker.py +work_dir: typing.Optional[str] = None -shared_cache: Cache = None +shared_cache: typing.Optional[Cache] = None class EnumerateError(Exception): """Non-retryable version enumeration error.""" -class DepsDevMixin: - """deps.dev mixin.""" - - _DEPS_DEV_ECOSYSTEM_MAP = { - 'Maven': 'MAVEN', - 'PyPI': 'PYPI', - } - - def _deps_dev_enumerate(self, - package, - introduced, - fixed=None, - last_affected=None, - limits=None): - """Use deps.dev to get list of versions.""" - ecosystem = self._DEPS_DEV_ECOSYSTEM_MAP[self.name] - url = _DEPS_DEV_API.format(ecosystem=ecosystem, package=package) - response = requests.get( - url, headers={ - 'X-DepsDev-APIKey': deps_dev_api_key, - }, timeout=TIMEOUT) - - if response.status_code == 404: - raise EnumerateError(f'Package {package} not found') - if response.status_code != 200: - raise RuntimeError( - f'Failed to get {ecosystem} versions for {package} with: ' - f'{response.text}') - - response = response.json() - versions = [v['version'] for v in response['versions']] - self.sort_versions(versions) - return self._get_affected_versions(versions, introduced, fixed, - last_affected, limits) - - -class Ecosystem: +class Ecosystem(ABC): """Ecosystem helpers.""" @property @@ -92,7 +66,7 @@ def name(self): return self.__class__.__name__ def _before_limits(self, version, limits): - """Return whether or not the given version is before any limits.""" + """Return whether the given version is before any limits.""" if not limits or '*' in limits: return True @@ -112,14 +86,15 @@ def next_version(self, package, version): return None + @abstractmethod def sort_key(self, version): """Sort key.""" - raise NotImplementedError def sort_versions(self, versions): """Sort versions.""" versions.sort(key=self.sort_key) + @abstractmethod def enumerate_versions(self, package, introduced, @@ -127,7 +102,6 @@ def enumerate_versions(self, last_affected=None, limits=None): """Enumerate versions.""" - raise NotImplementedError def _get_affected_versions(self, versions, introduced, fixed, last_affected, limits): @@ -171,6 +145,42 @@ def is_semver(self): return False +class DepsDevMixin(Ecosystem, ABC): + """deps.dev mixin.""" + + _DEPS_DEV_ECOSYSTEM_MAP = { + 'Maven': 'MAVEN', + 'PyPI': 'PYPI', + } + + def _deps_dev_enumerate(self, + package, + introduced, + fixed=None, + last_affected=None, + limits=None): + """Use deps.dev to get list of versions.""" + ecosystem = self._DEPS_DEV_ECOSYSTEM_MAP[self.name] + url = _DEPS_DEV_API.format(ecosystem=ecosystem, package=package) + response = requests.get( + url, headers={ + 'X-DepsDev-APIKey': deps_dev_api_key, + }, timeout=TIMEOUT) + + if response.status_code == 404: + raise EnumerateError(f'Package {package} not found') + if response.status_code != 200: + raise RuntimeError( + f'Failed to get {ecosystem} versions for {package} with: ' + f'{response.text}') + + response = response.json() + versions = [v['version'] for v in response['versions']] + self.sort_versions(versions) + return self._get_affected_versions(versions, introduced, fixed, + last_affected, limits) + + class SemverEcosystem(Ecosystem): """Generic semver ecosystem helpers.""" @@ -243,7 +253,7 @@ def enumerate_versions(self, last_affected, limits) -class Maven(Ecosystem, DepsDevMixin): +class Maven(DepsDevMixin): """Maven ecosystem.""" _API_PACKAGE_URL = 'https://search.maven.org/solrsearch/select' @@ -386,6 +396,98 @@ def enumerate_versions(self, last_affected, limits) +class Alpine(Ecosystem): + """Alpine packages ecosystem""" + + _APORTS_GIT_URL = 'https://gitlab.alpinelinux.org/alpine/aports.git' + _BRANCH_SUFFIX = '-stable' + alpine_release_ver: str + _GIT_REPO_PATH = 'version_enum/aports/' + + def __init__(self, alpine_release_ver: str): + self.alpine_release_ver = alpine_release_ver + + def get_branch_name(self) -> str: + return self.alpine_release_ver.lstrip('v') + self._BRANCH_SUFFIX + + def sort_key(self, version): + return AlpineLinuxVersion(version) + + @staticmethod + def _process_git_log(output: str) -> list: + """Takes git log diff output, + finds all changes to pkgver and outputs that in an unsorted list""" + all_versions = set() + lines = [ + x for x in output.splitlines() if len(x) == 0 or x.startswith('+pkgver') + ] + # Reverse so that it's in chronological order. + # The following loop also expects this order. + lines.reverse() + + current_ver = None + + for x in lines: + if len(x) == 0: + all_versions.add(current_ver) + continue + + x_split = x.split('=', 1) + if len(x_split) != 2: + # Skip the occasional invalid versions + continue + + ver = x_split[1] + if x.startswith('+pkgver'): + current_ver = ver + continue + + # Return unsorted list of versions + return list(all_versions) + + def enumerate_versions(self, + package: str, + introduced, + fixed=None, + last_affected=None, + limits=None): + if work_dir is None: + logging.error( + "Tried to enumerate alpine version without workdir being set") + return [] + + get_versions = self._get_versions + if shared_cache: + get_versions = cached(shared_cache)(get_versions) + + versions = get_versions(self.get_branch_name(), package) + self.sort_versions(versions) + + return self._get_affected_versions(versions, introduced, fixed, + last_affected, limits) + + @staticmethod + def _get_versions(branch: str, package: str) -> typing.List[str]: + """Get all versions for a package from aports repo""" + checkout_dir = os.path.join(work_dir, Alpine._GIT_REPO_PATH) + repos.ensure_updated_checkout( + Alpine._APORTS_GIT_URL, checkout_dir, branch=branch) + directories = glob.glob( + os.path.join(checkout_dir, '*', package.lower(), 'APKBUILD'), + recursive=True) + + if len(directories) != 1: + raise EnumerateError('Cannot find package in aports') + + relative_path = os.path.relpath(directories[0], checkout_dir) + stdout_data = subprocess.check_output( + ['git', 'log', '--oneline', '-L', '/pkgver=/,+2:' + relative_path], + cwd=checkout_dir).decode('utf-8') + + versions = Alpine._process_git_log(stdout_data) + return versions + + class Debian(Ecosystem): """Debian ecosystem""" @@ -512,10 +614,13 @@ def get(name: str) -> Ecosystem: if name.startswith('Debian:'): return Debian(name.split(':')[1]) + if name.startswith('Alpine:'): + return Alpine(name.split(':')[1]) + return _ecosystems.get(name) -def set_cache(cache: Cache): +def set_cache(cache: typing.Optional[Cache]): """Configures and enables the redis caching layer""" global shared_cache shared_cache = cache diff --git a/osv/ecosystems_test.py b/osv/ecosystems_test.py index 3f8e4c9bcfd..08734231e4e 100644 --- a/osv/ecosystems_test.py +++ b/osv/ecosystems_test.py @@ -19,6 +19,7 @@ import requests +from . import repos from . import cache from . import ecosystems @@ -103,28 +104,55 @@ def test_nuget(self): with self.assertRaises(ecosystems.EnumerateError): ecosystem.next_version('doesnotexist123456', '1') + @mock.patch( + 'osv.request_helper.requests.Session.get', + side_effect=requests.Session.get, + autospec=True) @mock.patch('osv.debian_version_cache.requests.get', side_effect=requests.get) - def test_debian(self, requests_mock: mock.MagicMock): - """Test Debian""" + def test_debian(self, first_ver_requests_mock: mock.MagicMock, + general_requests_mock: mock.MagicMock): + """Test Debian ecosystem enumeration and caching behaviour""" + in_memory_cache = cache.InMemoryCache() + ecosystems.set_cache(in_memory_cache) ecosystem = ecosystems.get('Debian:9') + + # Tests that next version and version enumeration generally works self.assertEqual('1.13.6-1', ecosystem.next_version('nginx', '1.13.5-1')) self.assertEqual('1.13.6-2', ecosystem.next_version('nginx', '1.13.6-1')) self.assertEqual('3.0.1+dfsg-2', ecosystem.next_version('blender', '3.0.1+dfsg-1')) + # Tests that sort key works self.assertGreater( ecosystem.sort_key('1.13.6-2'), ecosystem.sort_key('1.13.6-1')) + # Test that specifically is greater than normal versions self.assertGreater( ecosystem.sort_key(''), ecosystem.sort_key('1.13.6-1')) + # Test that end-of-life enumeration is disabled self.assertEqual( ecosystem.enumerate_versions('nginx', '0', ''), []) - # Test Ecosystem remover + + # Calls for first_version to the same ecosystem should be cached + self.assertEqual(first_ver_requests_mock.call_count, 1) + ecosystem.enumerate_versions('htop', '0') + + self.assertEqual(first_ver_requests_mock.call_count, 1) + + # Now start testing that Debian:10 contains different versions compared to 9 ecosystem = ecosystems.get('Debian:10') + + # Called 2 times so far, once for nginx, once for blender. + self.assertEqual(general_requests_mock.call_count, 3) # '0' as introduced version also tests the get_first_package_version func versions = ecosystem.enumerate_versions('cyrus-sasl2', '0', None) - self.assertEqual(requests_mock.call_count, 2) + self.assertEqual(general_requests_mock.call_count, 4) + + # new ecosystem, first version requests increase by 1 + self.assertEqual(first_ver_requests_mock.call_count, 2) + + # Check that only deb10 versions are in Debian:10, and no deb9 versions self.assertIn('2.1.27+dfsg-1+deb10u1', versions) self.assertNotIn('2.1.27~101-g0780600+dfsg-3+deb9u1', versions) self.assertNotIn('2.1.27~101-g0780600+dfsg-3+deb9u2', versions) @@ -132,9 +160,40 @@ def test_debian(self, requests_mock: mock.MagicMock): with self.assertRaises(ecosystems.EnumerateError): ecosystem.next_version('doesnotexist123456', '1') + self.assertEqual(general_requests_mock.call_count, 5) + # This should now only call the cache, and not requests.get ecosystem.enumerate_versions('cyrus-sasl2', '0', None) - self.assertEqual(requests_mock.call_count, 2) + self.assertEqual(first_ver_requests_mock.call_count, 2) + self.assertEqual(general_requests_mock.call_count, 5) + ecosystems.set_cache(None) + + @mock.patch( + 'osv.repos.ensure_updated_checkout', + side_effect=repos.ensure_updated_checkout) + def test_alpine(self, ensure_updated_checkout_mock: mock.MagicMock): + """Test Alpine ecosystem enumeration and caching behaviour""" + in_memory_cache = cache.InMemoryCache() + ecosystems.set_cache(in_memory_cache) + + # Set work_dir to allow cloning/fetching + ecosystems.work_dir = self._TEST_DATA_DIR + ecosystem = ecosystems.get('Alpine:v3.16') + self.assertEqual(ensure_updated_checkout_mock.call_count, 0) + # Tests that next version and version enumeration generally works + self.assertEqual('1.14.0', ecosystem.next_version('nginx', '1.12.2')) + self.assertEqual(ensure_updated_checkout_mock.call_count, 1) + self.assertEqual('1.16.1', ecosystem.next_version('nginx', '1.16.0')) + # Second call should use cache, so call count should not increase + self.assertEqual(ensure_updated_checkout_mock.call_count, 1) + + # Check letter suffixes clone correctly + self.assertEqual('2.78c', ecosystem.next_version('blender', '2.78a')) + + self.assertGreater( + ecosystem.sort_key('1.13.2'), ecosystem.sort_key('1.13.2_alpha')) + + ecosystems.set_cache(None) def test_packagist(self): """Test Packagist.""" diff --git a/osv/repos.py b/osv/repos.py index 3221a6264fc..00728832688 100644 --- a/osv/repos.py +++ b/osv/repos.py @@ -81,15 +81,21 @@ def _checkout_branch(repo, branch): repo.reset(remote_branch.target, pygit2.GIT_RESET_HARD) -def clone(git_url, checkout_dir, git_callbacks=None): - """Perform a clone.""" - # Use 'git' CLI here as it's much faster than libgit2's clone. +def _set_git_callback_env(git_callbacks): + """Set the environment variable to set git callbacks for cli git""" env = {} if git_callbacks: env['GIT_SSH_COMMAND'] = ( f'ssh -i "{git_callbacks.ssh_key_private_path}" ' f'-o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null ' f'-o User={git_callbacks.username} -o IdentitiesOnly=yes') + return env + + +def clone(git_url, checkout_dir, git_callbacks=None): + """Perform a clone.""" + # Use 'git' CLI here as it's much faster than libgit2's clone. + env = _set_git_callback_env(git_callbacks) subprocess.check_call( ['git', 'clone', _git_mirror(git_url), checkout_dir], @@ -158,7 +164,11 @@ def ensure_updated_checkout(git_url, def reset_repo(repo, git_callbacks): """Reset repo.""" - repo.remotes['origin'].fetch(callbacks=git_callbacks) + env = _set_git_callback_env(git_callbacks) + # Use git cli instead of pygit2 for performance + subprocess.check_call(['git', 'fetch', 'origin'], cwd=repo.workdir, env=env) + # Pygit2 equivalent of above call + # repo.remotes['origin'].fetch(callbacks=git_callbacks) remote_branch = repo.lookup_branch( repo.head.name.replace('refs/heads/', 'origin/'), pygit2.GIT_BRANCH_REMOTE) diff --git a/osv/testdata/.gitignore b/osv/testdata/.gitignore new file mode 100644 index 00000000000..f7bc0fe8b7c --- /dev/null +++ b/osv/testdata/.gitignore @@ -0,0 +1 @@ +version_enum \ No newline at end of file diff --git a/osv/third_party/univers/alpine.py b/osv/third_party/univers/alpine.py new file mode 100644 index 00000000000..fb69334ffbe --- /dev/null +++ b/osv/third_party/univers/alpine.py @@ -0,0 +1,159 @@ +# +# Copyright (c) nexB Inc. and others. +# SPDX-License-Identifier: Apache-2.0 +# +# Visit https://aboutcode.org and +# https://github.com/nexB/univers for support and download. +import attr + +from . import gentoo +from .utils import remove_spaces + + +class InvalidVersion(ValueError): + pass + + +def is_valid_alpine_version(s): + """ + Return True is the string `s` is a valid Alpine version. + We do not support yet version strings that start with + non-significant zeros. + For example: + >>> is_valid_alpine_version("006") + False + >>> is_valid_alpine_version("1.2.3") + True + >>> is_valid_alpine_version("02-r1") + False + """ + left, _, _ = s.partition(".") + # hanlde the suffix case + left, _, _ = left.partition("-") + if not left.isdigit(): + return True + i = int(left) + return str(i) == left + + +@attr.s(frozen=True, order=False, hash=True) +class Version: + """ + Base version mixin to subclass for each version syntax implementation. + Each version subclass is: + - immutable and hashable + - comparable and orderable e.g., such as implementing all rich comparison + operators or implementing functools.total_ordering. The default is to + compare the value as-is. + """ + + # the original string used to build this Version + string = attr.ib(type=str) + + # the normalized string for this Version, stored without spaces and + # lowercased. Any leading v is removed too. + normalized_string = attr.ib(type=str, default=None, repr=False) + + # a comparable scheme-specific version object constructed from + # the version string + value = attr.ib(default=None, repr=False) + + def __attrs_post_init__(self): + normalized_string = self.normalize(self.string) + if not self.is_valid(normalized_string): + raise InvalidVersion(f"{self.string!r} is not a valid {self.__class__!r}") + + # Set the normalized string as default value + + # Notes: setattr is used because this is an immutable frozen instance. + # See https://www.attrs.org/en/stable/init.html?#post-init + object.__setattr__(self, "normalized_string", normalized_string) + value = self.build_value(normalized_string) + object.__setattr__(self, "value", value) + + @classmethod + def is_valid(cls, string): + """ + Return True if the ``string`` is a valid version for its scheme or False + if not valid. The empty string, None, False and 0 are considered invalid. + Subclasses should implement this. + """ + return bool(string) + + @classmethod + def normalize(cls, string): + """ + Return a normalized version string from ``string ``. Subclass can override. + """ + # FIXME: Is removing spaces and strip v the right thing to do? + return remove_spaces(string).rstrip("v ").strip() + + @classmethod + def build_value(cls, string): + """ + Return a wrapped version "value" object for a version ``string``. + Subclasses can override. The default is a no-op and returns the string + as-is, and is called by default at init time with the computed + normalized_string. + """ + return string + + def satisfies(self, constraint): + """ + Return True is this Version satisfies the ``constraint`` + VersionConstraint. Satisfying means that this version is "within" the + ``constraint``. + """ + return self in constraint + + def __str__(self): + return str(self.value) + + def __eq__(self, other): + if not isinstance(other, self.__class__): + return NotImplemented + return self.value.__eq__(other.value) + + def __lt__(self, other): + if not isinstance(other, self.__class__): + return NotImplemented + return self.value.__lt__(other.value) + + def __gt__(self, other): + if not isinstance(other, self.__class__): + return NotImplemented + return self.value.__gt__(other.value) + + def __le__(self, other): + if not isinstance(other, self.__class__): + return NotImplemented + return self.value.__le__(other.value) + + def __ge__(self, other): + if not isinstance(other, self.__class__): + return NotImplemented + return self.value.__ge__(other.value) + + +@attr.s(frozen=True, order=False, eq=False, hash=True) +class AlpineLinuxVersion(Version): + """Alpine linux version""" + + @classmethod + def is_valid(cls, string): + return is_valid_alpine_version(string) and gentoo.is_valid(string) + + def __eq__(self, other): + if not isinstance(other, self.__class__): + return NotImplemented + return gentoo.vercmp(self.value, other.value) == 0 + + def __lt__(self, other): + if not isinstance(other, self.__class__): + return NotImplemented + return gentoo.vercmp(self.value, other.value) < 0 + + def __gt__(self, other): + if not isinstance(other, self.__class__): + return NotImplemented + return gentoo.vercmp(self.value, other.value) > 0 diff --git a/osv/third_party/univers/gentoo.py b/osv/third_party/univers/gentoo.py new file mode 100644 index 00000000000..89a88fe7e65 --- /dev/null +++ b/osv/third_party/univers/gentoo.py @@ -0,0 +1,181 @@ +# +# Copyright (c) 2006-2019, pkgcore contributors +# SPDX-License-Identifier: BSD-3-Clause +# Version comparison utility extracted from pkgcore and further stripped down. +# +# Visit https://aboutcode.org and https://github.com/nexB/univers for support and download. + +import re + +from .utils import cmp +from .utils import remove_spaces + +_is_gentoo_version = re.compile( + r"^(?:\d+)(?:\.\d+)*[a-zA-Z]?(?:_(p(?:re)?|beta|alpha|rc)\d*)*$").match + +suffix_regexp = re.compile("^(alpha|beta|rc|pre|p)(\\d*)$") + +revision_regexp = re.compile(r".*(-r\d+)") + +suffix_value = {"pre": -2, "p": 1, "alpha": -4, "beta": -3, "rc": -1} +""" +gentoo ebuild version comparison +""" + + +def is_valid(string): + version, _ = parse_version_and_revision(remove_spaces(string)) + return _is_gentoo_version(version) + + +def parse_version_and_revision(version_string): + """ + Return a tuple of (version string, revision int) given a ``version_string``. + """ + revision = 0 + version = version_string + match = revision_regexp.search(version_string) + if match: + revision = int(match.group(1)[2:]) + version = version_string[:match.span(1)[0]] + + return version, revision + + +def vercmp(ver1, ver2): + """ + Compare two versions ``ver1`` and ``ver2`` and return 0, 1, or -1 according + to the Python 2 cmp() semantics: + + Compare the two objects x and y and return an integer according to the + outcome. The return value is negative if x < y, zero if x == y and + strictly positive if x > y. + """ + if not ver1: + if not ver2: + return 0 + else: + return -1 + elif not ver2: + return 1 + + ver1, rev1 = parse_version_and_revision(ver1) + ver2, rev2 = parse_version_and_revision(ver2) + + # If the versions are the same, comparing revisions will suffice. + if ver1 == ver2: + # revisions are equal if 0 or None (versionless cpv) + if not rev1 and not rev2: + return 0 + return cmp(rev1, rev2) + + # Split up the versions into dotted strings and lists of suffixes. + parts1 = ver1.split("_") + parts2 = ver2.split("_") + + # If the dotted strings are equal, we can skip doing a detailed comparison. + if parts1[0] != parts2[0]: + + # First split up the dotted strings into their components. + ver_parts1 = parts1[0].split(".") + ver_parts2 = parts2[0].split(".") + + # Pull out any letter suffix on the final components and keep + # them for later. + letters = [] + for ver_parts in (ver_parts1, ver_parts2): + if ver_parts[-1][-1].isalpha(): + letters.append(ord(ver_parts[-1][-1])) + ver_parts[-1] = ver_parts[-1][:-1] + else: + # Using -1 simplifies comparisons later + letters.append(-1) + + # OPT: Pull length calculation out of the loop + ver_parts1_len = len(ver_parts1) + ver_parts2_len = len(ver_parts2) + + # Iterate through the components + for v1, v2 in zip(ver_parts1, ver_parts2): + + # If the string components are equal, the numerical + # components will be equal too. + if v1 == v2: + continue + + # If one of the components begins with a "0" then they + # are compared as floats so that 1.1 > 1.02; else ints. + if v1[0] != "0" and v2[0] != "0": + v1 = int(v1) + v2 = int(v2) + else: + # handle the 0.060 == 0.060 case. + v1 = v1.rstrip("0") + v2 = v2.rstrip("0") + + # If they are not equal, the higher value wins. + c = cmp(v1, v2) + if c: + return c + + if ver_parts1_len > ver_parts2_len: + return 1 + elif ver_parts2_len > ver_parts1_len: + return -1 + + # The dotted components were equal. Let's compare any single + # letter suffixes. + if letters[0] != letters[1]: + return cmp(letters[0], letters[1]) + + # The dotted components were equal, so remove them from our lists + # leaving only suffixes. + del parts1[0] + del parts2[0] + + # OPT: Pull length calculation out of the loop + parts1_len = len(parts1) + parts2_len = len(parts2) + + # Iterate through the suffixes + for x in range(max(parts1_len, parts2_len)): + + # If we're at the end of one of our lists, we need to use + # the next suffix from the other list to decide who wins. + if x == parts1_len: + match = suffix_regexp.match(parts2[x]) + val = suffix_value[match.group(1)] + if val: + return cmp(0, val) + return cmp(0, int("0" + match.group(2))) + if x == parts2_len: + match = suffix_regexp.match(parts1[x]) + val = suffix_value[match.group(1)] + if val: + return cmp(val, 0) + return cmp(int("0" + match.group(2)), 0) + + # If the string values are equal, no need to parse them. + # Continue on to the next. + if parts1[x] == parts2[x]: + continue + + # Match against our regular expression to make a split between + # "beta" and "1" in "beta1" + match1 = suffix_regexp.match(parts1[x]) + match2 = suffix_regexp.match(parts2[x]) + + # If our int'ified suffix names are different, use that as the basis + # for comparison. + c = cmp(suffix_value[match1.group(1)], suffix_value[match2.group(1)]) + if c: + return c + + # Otherwise use the digit as the basis for comparison. + c = cmp(int("0" + match1.group(2)), int("0" + match2.group(2))) + if c: + return c + + # Our versions had different strings but ended up being equal. + # The revision holds the final difference. + return cmp(rev1, rev2) diff --git a/osv/third_party/univers/gentoo.py.ABOUT b/osv/third_party/univers/gentoo.py.ABOUT new file mode 100644 index 00000000000..3e76a2c6c0b --- /dev/null +++ b/osv/third_party/univers/gentoo.py.ABOUT @@ -0,0 +1,11 @@ +about_resource: gentoo.py +package_url: pkg:pypi/pkgcore@0.11.8 +copyright: | + Copyright (c) 2006-2019, pkgcore contributors + +license_expression: BSD-3-Clause +homepage_url: https://github.com/pkgcore/pkgcore/blob/master/src/pkgcore/ebuild/cpv.py + +notes: The version comparison utility is extracted from pkgcore and further stripped down. + +notice_file: gentoo.py.NOTICE \ No newline at end of file diff --git a/osv/third_party/univers/gentoo.py.NOTICE b/osv/third_party/univers/gentoo.py.NOTICE new file mode 100644 index 00000000000..fbcbbdb9314 --- /dev/null +++ b/osv/third_party/univers/gentoo.py.NOTICE @@ -0,0 +1,23 @@ +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +1. Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. +2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. +3. Neither the name of pkgcore nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. \ No newline at end of file diff --git a/osv/third_party/univers/utils.py b/osv/third_party/univers/utils.py new file mode 100644 index 00000000000..a657c3951f9 --- /dev/null +++ b/osv/third_party/univers/utils.py @@ -0,0 +1,29 @@ +# +# Copyright (c) nexB Inc. and others. +# SPDX-License-Identifier: Apache-2.0 +# +# Visit https://aboutcode.org and https://github.com/nexB/univers for support and download. + + +def remove_spaces(string): + return "".join(string.split()) + + +def cmp(x, y): + """ + Replacement for built-in Python 2 function cmp that was removed in Python 3 + From https://docs.python.org/2/library/functions.html?highlight=cmp#cmp : + + Compare the two objects x and y and return an integer according to the + outcome. The return value is negative if x < y, zero if x == y and + strictly positive if x > y. + """ + if x == y: + return 0 + elif x is None: + return -1 + elif y is None: + return 1 + else: + # note that this is the minimal replacement function + return (x > y) - (x < y)