actions/component-detection-dependency-submission-action
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Update parsing logic

Browse Source
This commit is contained in:
Justin Hutchings
2023-01-20 00:02:28 +00:00
parent 6de9c7319f
commit e901ca3883
2 changed files with 221 additions and 59 deletions
Download Patch File Download Diff File Expand all files Collapse all files
condaParser.test.ts
+129 -2
View File
@@ -5,8 +5,135 @@ test('Gets files', async () => {
expect(files.length).toEqual(1);
});
test('Parses manifests', async() => {
function roundTripJSON(obj: any): object {
return JSON.parse(JSON.stringify(obj))
}
test('Parses manifests', async () => {
var files = conda.searchFiles("test", "environment.yaml");
var manifests = conda.getManifestsFromEnvironmentFiles(files);
expect(manifests.length).toEqual(1);
})
expect(roundTripJSON(manifests[0])).toEqual(
{
"resolved": {
"pkg:conda/python@3.8": {
"package_url": "pkg:conda/python@3.8",
"relationship": "direct",
"dependencies": []
},
"pkg:conda/pytorch@1.10": {
"package_url": "pkg:conda/pytorch@1.10",
"relationship": "direct",
"dependencies": []
},
"pkg:conda/torchvision": {
"package_url": "pkg:conda/torchvision",
"relationship": "direct",
"dependencies": []
},
"pkg:conda/cudatoolkit@11.0": {
"package_url": "pkg:conda/cudatoolkit@11.0",
"relationship": "direct",
"dependencies": []
},
"pkg:conda/pip": {
"package_url": "pkg:conda/pip",
"relationship": "direct",
"dependencies": []
},
"pkg:pypi/pytorch-lightning@1.5.2": {
"package_url": "pkg:pypi/pytorch-lightning@1.5.2",
"relationship": "direct",
"dependencies": []
}, "pkg:pypi/einops@0.3.2": {
"package_url": "pkg:pypi/einops@0.3.2",
"relationship": "direct",
"dependencies": []
},
"pkg:pypi/kornia@0.6.1": {
"package_url": "pkg:pypi/kornia@0.6.1",
"relationship": "direct",
"dependencies": []
},
"pkg:pypi/opencv-python@4.5.4.58": {
"package_url": "pkg:pypi/opencv-python@4.5.4.58",
"relationship": "direct",
"dependencies": []
},
"pkg:pypi/matplotlib@3.5.0": {
"package_url": "pkg:pypi/matplotlib@3.5.0",
"relationship": "direct",
"dependencies": []
},
"pkg:pypi/imageio@2.10.4": {
"package_url": "pkg:pypi/imageio@2.10.4",
"relationship": "direct",
"dependencies": []
},
"pkg:pypi/imageio-ffmpeg@0.4.5": {
"package_url": "pkg:pypi/imageio-ffmpeg@0.4.5",
"relationship": "direct",
"dependencies": []
},
"pkg:pypi/torch-optimizer@0.3.0": {
"package_url": "pkg:pypi/torch-optimizer@0.3.0",
"relationship": "direct",
"dependencies": []
},
"pkg:pypi/setuptools@58.2.0": {
"package_url": "pkg:pypi/setuptools@58.2.0",
"relationship": "direct",
"dependencies": []
},
"pkg:pypi/pymcubes@0.1.2": {
"package_url": "pkg:pypi/pymcubes@0.1.2",
"relationship": "direct",
"dependencies": []
},
"pkg:pypi/pycollada@0.7.1": {
"package_url": "pkg:pypi/pycollada@0.7.1",
"relationship": "direct",
"dependencies": []
},
"pkg:pypi/trimesh@3.9.1": {
"package_url": "pkg:pypi/trimesh@3.9.1",
"relationship": "direct",
"dependencies": []
},
"pkg:pypi/pyglet@1.5.10": {
"package_url": "pkg:pypi/pyglet@1.5.10"
, "relationship": "direct",
"dependencies": []
},
"pkg:pypi/networkx@2.5": {
"package_url": "pkg:pypi/networkx@2.5",
"relationship": "direct",
"dependencies": []
},
"pkg:pypi/plyfile@0.7.2": {
"package_url": "pkg:pypi/plyfile@0.7.2",
"relationship": "direct",
"dependencies": []
},
"pkg:pypi/open3d@0.13.0": {
"package_url": "pkg:pypi/open3d@0.13.0",
"relationship": "direct",
"dependencies": []
},
"pkg:pypi/configargparse@1.5.3": {
"package_url": "pkg:pypi/configargparse@1.5.3",
"relationship": "direct",
"dependencies": []
},
"pkg:pypi/ninja": {
"package_url": "pkg:pypi/ninja",
"relationship": "direct",
"dependencies": []
}
},
"name": "test",
"file": {
"source_location": "test/environment.yaml"
}
})
});
condaParser.ts
+92 -57
View File
@@ -14,73 +14,108 @@ import {
} from '@github/dependency-submission-toolkit'
import { YAMLMap } from 'yaml';
/**getManifestFromEnvironmentFile(document, fileName) {
core.debug(`getManifestFromEnvironmentFile processing ${fileName}`);
/**getManifestFromEnvironmentFile(document, fileName) {
core.debug(`getManifestFromEnvironmentFile processing ${fileName}`);
let manifest = new Manifest("Environment", fileName);
let manifest = new Manifest("Environment", fileName);
/**
let manifest = new Manifest(document.name, fileName);
/**
let manifest = new Manifest(document.name, fileName);
core.debug(`Processing ${document.packages?.length} packages`);
core.debug(`Processing ${document.packages?.length} packages`);
document.packages?.forEach(pkg => {
let packageName = pkg.name;
let packageVersion = pkg.packageVersion;
let referenceLocator = pkg.externalRefs?.find(ref => ref.referenceCategory === "PACKAGE-MANAGER" && ref.referenceType === "purl")?.referenceLocator;
let genericPurl = `pkg:generic/${packageName}@${packageVersion}`;
// SPDX 2.3 defines a purl field
let purl;
if (pkg.purl != undefined) {
purl = pkg.purl;
} else if (referenceLocator != undefined) {
purl = referenceLocator;
} else {
purl = genericPurl;
}
document.packages?.forEach(pkg => {
let packageName = pkg.name;
let packageVersion = pkg.packageVersion;
let referenceLocator = pkg.externalRefs?.find(ref => ref.referenceCategory === "PACKAGE-MANAGER" && ref.referenceType === "purl")?.referenceLocator;
let genericPurl = `pkg:generic/${packageName}@${packageVersion}`;
// SPDX 2.3 defines a purl field
let purl;
if (pkg.purl != undefined) {
purl = pkg.purl;
} else if (referenceLocator != undefined) {
purl = referenceLocator;
} else {
purl = genericPurl;
}
// Working around weird encoding issues from an SBOM generator
// Find the last instance of %40 and replace it with @
purl = replaceVersionEscape(purl);
// Working around weird encoding issues from an SBOM generator
// Find the last instance of %40 and replace it with @
purl = replaceVersionEscape(purl);
let relationships = document.relationships?.find(rel => rel.relatedSpdxElement == pkg.SPDXID && rel.relationshipType == "DEPENDS_ON" && rel.spdxElementId != "SPDXRef-RootPackage");
if (relationships != null && relationships.length > 0) {
manifest.addIndirectDependency(new Package(purl));
} else {
let relationships = document.relationships?.find(rel => rel.relatedSpdxElement == pkg.SPDXID && rel.relationshipType == "DEPENDS_ON" && rel.spdxElementId != "SPDXRef-RootPackage");
if (relationships != null && relationships.length > 0) {
manifest.addIndirectDependency(new Package(purl));
} else {
manifest.addDirectDependency(new Package(purl));
}
});
return manifest;
}*/
/***/
export default class CondaParser {
static searchFiles(filePath = "", filePattern = "") {
if (filePath == "") {
let filePath = core.getInput('filePath');
}
if (filePattern == "") {
let filePattern = core.getInput('filePattern');
}
return glob.sync(`${filePath}/${filePattern}`, {});
}
static getManifestsFromEnvironmentFiles(files: string[]) {
core.debug(`Processing ${files.length} files`);
let manifests: any[] = [];
files?.forEach(filePath => {
core.debug(`Processing ${filePath}`);
const contents = fs.readFileSync(filePath, 'utf8')
manifests.push(CondaParser.getManifestFromYaml(yaml.parse(contents), filePath));
});
return manifests;
}
// Gets a Manifest object from an environment.yaml
static getManifestFromYaml(yaml: any, filePath: string) {
core.debug(`getManifestFromEnvironmentFile processing ${yaml}`);
let manifest = new Manifest(yaml.name, filePath);
yaml.dependencies?.forEach((dependency: any) => {
let purl = "";
// If it's an object with the collection `pip`, then these are PyPI dependencies
if (dependency instanceof Object && dependency.pip != null) {
dependency.pip.forEach((pipDependency:string) => {
purl = this.getPurlFromDependency(pipDependency, "pypi");
manifest.addDirectDependency(new Package(purl));
});
} else {
purl = this.getPurlFromDependency(dependency, "conda");
manifest.addDirectDependency(new Package(purl));
}
}
});
return manifest;
}*/
}
// Gets a purl string from an environment file's list of dependencies
static getPurlFromDependency(dependency: string, ecosystem: string) {
// Versions look like "python=3.8" or if nested in a pip section like "pytorch==1.0"
// Split on the '=' to separate the packageName and version
let delimiter = (ecosystem == "pypi") ? "==" : "=";
let split = dependency.split(delimiter);
let packageName = split[0];
// If there is a version specified, use it, otherwise, leave it off.
let version = (split.length > 1) ? `@${split[1]}` : "";
/***/
// If it's a PyPI dependency, then normalize the package name
if (ecosystem == "pypi") {
packageName = packageName.toLowerCase().replace("_", "-");
}
export default class CondaParser {
static searchFiles(filePath = "", filePattern = "") {
if (filePath == "") {
let filePath = core.getInput('filePath');
}
if (filePattern == "") {
let filePattern = core.getInput('filePattern');
}
return glob.sync(`${filePath}/${filePattern}`, {});
}
static getManifestsFromEnvironmentFiles(files:string[]) {
core.debug(`Processing ${files.length} files`);
let manifests: any[] = [];
files?.forEach(filePath => {
core.debug(`Processing ${filePath}`);
const contents = fs.readFileSync(filePath, 'utf8')
manifests.push(yaml.parse(contents));
});
return manifests;
}
static getManifestFromYaml(yaml:any) {
}
return `pkg:${ecosystem}/${packageName}${version}`;
}
}