From e901ca38835cd61455e3a55bdd93b64fc1875e0c Mon Sep 17 00:00:00 2001 From: Justin Hutchings Date: Fri, 20 Jan 2023 00:02:28 +0000 Subject: [PATCH] Update parsing logic --- condaParser.test.ts | 131 +++++++++++++++++++++++++++++++++++++- condaParser.ts | 149 +++++++++++++++++++++++++++----------------- 2 files changed, 221 insertions(+), 59 deletions(-) diff --git a/condaParser.test.ts b/condaParser.test.ts index 8943b99..5d9d2ad 100644 --- a/condaParser.test.ts +++ b/condaParser.test.ts @@ -5,8 +5,135 @@ test('Gets files', async () => { expect(files.length).toEqual(1); }); -test('Parses manifests', async() => { +function roundTripJSON(obj: any): object { + return JSON.parse(JSON.stringify(obj)) +} + +test('Parses manifests', async () => { var files = conda.searchFiles("test", "environment.yaml"); var manifests = conda.getManifestsFromEnvironmentFiles(files); expect(manifests.length).toEqual(1); -}) \ No newline at end of file + expect(roundTripJSON(manifests[0])).toEqual( + { + "resolved": { + "pkg:conda/python@3.8": { + "package_url": "pkg:conda/python@3.8", + "relationship": "direct", + "dependencies": [] + }, + "pkg:conda/pytorch@1.10": { + "package_url": "pkg:conda/pytorch@1.10", + "relationship": "direct", + "dependencies": [] + }, + "pkg:conda/torchvision": { + "package_url": "pkg:conda/torchvision", + "relationship": "direct", + "dependencies": [] + }, + "pkg:conda/cudatoolkit@11.0": { + "package_url": "pkg:conda/cudatoolkit@11.0", + "relationship": "direct", + "dependencies": [] + }, + "pkg:conda/pip": { + "package_url": "pkg:conda/pip", + "relationship": "direct", + "dependencies": [] + }, + "pkg:pypi/pytorch-lightning@1.5.2": { + "package_url": "pkg:pypi/pytorch-lightning@1.5.2", + "relationship": "direct", + "dependencies": [] + }, "pkg:pypi/einops@0.3.2": { + "package_url": "pkg:pypi/einops@0.3.2", + "relationship": "direct", + "dependencies": [] + }, + "pkg:pypi/kornia@0.6.1": { + "package_url": "pkg:pypi/kornia@0.6.1", + "relationship": "direct", + "dependencies": [] + }, + "pkg:pypi/opencv-python@4.5.4.58": { + "package_url": "pkg:pypi/opencv-python@4.5.4.58", + "relationship": "direct", + "dependencies": [] + }, + "pkg:pypi/matplotlib@3.5.0": { + "package_url": "pkg:pypi/matplotlib@3.5.0", + "relationship": "direct", + "dependencies": [] + }, + "pkg:pypi/imageio@2.10.4": { + "package_url": "pkg:pypi/imageio@2.10.4", + "relationship": "direct", + "dependencies": [] + }, + "pkg:pypi/imageio-ffmpeg@0.4.5": { + "package_url": "pkg:pypi/imageio-ffmpeg@0.4.5", + "relationship": "direct", + "dependencies": [] + }, + "pkg:pypi/torch-optimizer@0.3.0": { + "package_url": "pkg:pypi/torch-optimizer@0.3.0", + "relationship": "direct", + "dependencies": [] + }, + "pkg:pypi/setuptools@58.2.0": { + "package_url": "pkg:pypi/setuptools@58.2.0", + "relationship": "direct", + "dependencies": [] + }, + "pkg:pypi/pymcubes@0.1.2": { + "package_url": "pkg:pypi/pymcubes@0.1.2", + "relationship": "direct", + "dependencies": [] + }, + "pkg:pypi/pycollada@0.7.1": { + "package_url": "pkg:pypi/pycollada@0.7.1", + "relationship": "direct", + "dependencies": [] + }, + "pkg:pypi/trimesh@3.9.1": { + "package_url": "pkg:pypi/trimesh@3.9.1", + "relationship": "direct", + "dependencies": [] + }, + "pkg:pypi/pyglet@1.5.10": { + "package_url": "pkg:pypi/pyglet@1.5.10" + , "relationship": "direct", + "dependencies": [] + }, + "pkg:pypi/networkx@2.5": { + "package_url": "pkg:pypi/networkx@2.5", + "relationship": "direct", + "dependencies": [] + }, + "pkg:pypi/plyfile@0.7.2": { + "package_url": "pkg:pypi/plyfile@0.7.2", + "relationship": "direct", + "dependencies": [] + }, + "pkg:pypi/open3d@0.13.0": { + "package_url": "pkg:pypi/open3d@0.13.0", + "relationship": "direct", + "dependencies": [] + }, + "pkg:pypi/configargparse@1.5.3": { + "package_url": "pkg:pypi/configargparse@1.5.3", + "relationship": "direct", + "dependencies": [] + }, + "pkg:pypi/ninja": { + "package_url": "pkg:pypi/ninja", + "relationship": "direct", + "dependencies": [] + } + }, + "name": "test", + "file": { + "source_location": "test/environment.yaml" + } + }) + }); \ No newline at end of file diff --git a/condaParser.ts b/condaParser.ts index 6cd5da8..5900671 100644 --- a/condaParser.ts +++ b/condaParser.ts @@ -14,73 +14,108 @@ import { } from '@github/dependency-submission-toolkit' import { YAMLMap } from 'yaml'; - /**getManifestFromEnvironmentFile(document, fileName) { - core.debug(`getManifestFromEnvironmentFile processing ${fileName}`); +/**getManifestFromEnvironmentFile(document, fileName) { +core.debug(`getManifestFromEnvironmentFile processing ${fileName}`); - let manifest = new Manifest("Environment", fileName); +let manifest = new Manifest("Environment", fileName); - /** - let manifest = new Manifest(document.name, fileName); +/** + let manifest = new Manifest(document.name, fileName); - core.debug(`Processing ${document.packages?.length} packages`); +core.debug(`Processing ${document.packages?.length} packages`); - document.packages?.forEach(pkg => { - let packageName = pkg.name; - let packageVersion = pkg.packageVersion; - let referenceLocator = pkg.externalRefs?.find(ref => ref.referenceCategory === "PACKAGE-MANAGER" && ref.referenceType === "purl")?.referenceLocator; - let genericPurl = `pkg:generic/${packageName}@${packageVersion}`; - // SPDX 2.3 defines a purl field - let purl; - if (pkg.purl != undefined) { - purl = pkg.purl; - } else if (referenceLocator != undefined) { - purl = referenceLocator; - } else { - purl = genericPurl; - } +document.packages?.forEach(pkg => { + let packageName = pkg.name; + let packageVersion = pkg.packageVersion; + let referenceLocator = pkg.externalRefs?.find(ref => ref.referenceCategory === "PACKAGE-MANAGER" && ref.referenceType === "purl")?.referenceLocator; + let genericPurl = `pkg:generic/${packageName}@${packageVersion}`; + // SPDX 2.3 defines a purl field + let purl; + if (pkg.purl != undefined) { + purl = pkg.purl; + } else if (referenceLocator != undefined) { + purl = referenceLocator; + } else { + purl = genericPurl; + } - // Working around weird encoding issues from an SBOM generator - // Find the last instance of %40 and replace it with @ - purl = replaceVersionEscape(purl); + // Working around weird encoding issues from an SBOM generator + // Find the last instance of %40 and replace it with @ + purl = replaceVersionEscape(purl); - let relationships = document.relationships?.find(rel => rel.relatedSpdxElement == pkg.SPDXID && rel.relationshipType == "DEPENDS_ON" && rel.spdxElementId != "SPDXRef-RootPackage"); - if (relationships != null && relationships.length > 0) { - manifest.addIndirectDependency(new Package(purl)); - } else { + let relationships = document.relationships?.find(rel => rel.relatedSpdxElement == pkg.SPDXID && rel.relationshipType == "DEPENDS_ON" && rel.spdxElementId != "SPDXRef-RootPackage"); + if (relationships != null && relationships.length > 0) { + manifest.addIndirectDependency(new Package(purl)); + } else { + manifest.addDirectDependency(new Package(purl)); + } +}); +return manifest; +}*/ + +/***/ + +export default class CondaParser { + + static searchFiles(filePath = "", filePattern = "") { + if (filePath == "") { + let filePath = core.getInput('filePath'); + } + if (filePattern == "") { + let filePattern = core.getInput('filePattern'); + } + + return glob.sync(`${filePath}/${filePattern}`, {}); + } + + static getManifestsFromEnvironmentFiles(files: string[]) { + core.debug(`Processing ${files.length} files`); + let manifests: any[] = []; + files?.forEach(filePath => { + core.debug(`Processing ${filePath}`); + const contents = fs.readFileSync(filePath, 'utf8') + manifests.push(CondaParser.getManifestFromYaml(yaml.parse(contents), filePath)); + }); + return manifests; + } + + // Gets a Manifest object from an environment.yaml + static getManifestFromYaml(yaml: any, filePath: string) { + core.debug(`getManifestFromEnvironmentFile processing ${yaml}`); + + let manifest = new Manifest(yaml.name, filePath); + yaml.dependencies?.forEach((dependency: any) => { + let purl = ""; + // If it's an object with the collection `pip`, then these are PyPI dependencies + if (dependency instanceof Object && dependency.pip != null) { + dependency.pip.forEach((pipDependency:string) => { + purl = this.getPurlFromDependency(pipDependency, "pypi"); + manifest.addDirectDependency(new Package(purl)); + }); + } else { + purl = this.getPurlFromDependency(dependency, "conda"); manifest.addDirectDependency(new Package(purl)); - } + } }); return manifest; - }*/ + } + + // Gets a purl string from an environment file's list of dependencies + static getPurlFromDependency(dependency: string, ecosystem: string) { + // Versions look like "python=3.8" or if nested in a pip section like "pytorch==1.0" + // Split on the '=' to separate the packageName and version + let delimiter = (ecosystem == "pypi") ? "==" : "="; + let split = dependency.split(delimiter); + let packageName = split[0]; + // If there is a version specified, use it, otherwise, leave it off. + let version = (split.length > 1) ? `@${split[1]}` : ""; - /***/ + // If it's a PyPI dependency, then normalize the package name + if (ecosystem == "pypi") { + packageName = packageName.toLowerCase().replace("_", "-"); + } - export default class CondaParser { - - static searchFiles(filePath = "", filePattern = "") { - if (filePath == "") { - let filePath = core.getInput('filePath'); - } - if (filePattern == "") { - let filePattern = core.getInput('filePattern'); - } - - return glob.sync(`${filePath}/${filePattern}`, {}); - } - - static getManifestsFromEnvironmentFiles(files:string[]) { - core.debug(`Processing ${files.length} files`); - let manifests: any[] = []; - files?.forEach(filePath => { - core.debug(`Processing ${filePath}`); - const contents = fs.readFileSync(filePath, 'utf8') - manifests.push(yaml.parse(contents)); - }); - return manifests; - } - - static getManifestFromYaml(yaml:any) { - - } + return `pkg:${ecosystem}/${packageName}${version}`; + } } \ No newline at end of file