refactor(vulnfeeds): move converters/mirrors into subdirectories for cleaner understanding

vulnfeeds/cmd/combine-to-osv/README.md

@@ -9,14 +9,14 @@ Combine [`PackageInfo`](https://github.com/google/osv.dev/blob/2c22e9534a521c6c6
 To address the generation of CVE records from multiple disparate sources (all requiring a common record prefix):
 
 * Alpine, by [this code](../alpine)
-* the NVD, by [this code](../nvd-cve-osv)
+* the NVD, by [this code](../converters/cve/nvd-cve-osv)
 
 ## How
 
 See [`run_combine_to_osv_convert.sh`](run_combine_to_osv_convert.sh):
 
 * Reads from [`gs://cve-osv-conversion/parts`](https://storage.googleapis.com/cve-osv-conversion/index.html?prefix=parts/)
-* Merges with CVE data from NVD (obtained from GCS mirror maintained by [`download-cves`](../download-cves/mirror_nvd.sh))
+* Merges with CVE data from NVD (obtained from GCS mirror maintained by [`download-cves`](../mirrors/download-cves/mirror_nvd.sh))
 * Writes an OSV record to [`gs://cve-osv-conversion/osv-output`](https://storage.googleapis.com/cve-osv-conversion/index.html?prefix=osv-output/)
   * This is the import source for [`cve-osv`](https://github.com/google/osv.dev/blob/2c22e9534a521c6c6350275427f80e481065ca39/source.yaml#L96)
   * What gets written can be overridden by OSV records in [`gs://cve-osv-conversion/osv-output-overrides`](https://storage.googleapis.com/cve-osv-conversion/index.html?prefix=osv-output-overrides/)

vulnfeeds/cmd/alpine/Dockerfile → vulnfeeds/cmd/converters/alpine/Dockerfile

@@ -22,13 +22,13 @@ COPY ./go.sum /src/go.sum
 RUN go mod download
 
 COPY ./ /src/
-RUN go build -o alpine-osv ./cmd/alpine/
+RUN go build -o alpine-osv ./cmd/converters/alpine/
 
 
 FROM gcr.io/google.com/cloudsdktool/google-cloud-cli:alpine@sha256:feca5d4cb9b422e124e6f28b8ed2e714160757eb383eaae712117c75f584aa2f
 
 WORKDIR /root/
 COPY --from=GO_BUILD /src/alpine-osv ./
-COPY ./cmd/alpine/run_alpine_convert.sh ./
+COPY ./cmd/converters/alpine/run_alpine_convert.sh ./
 
 ENTRYPOINT ["/root/run_alpine_convert.sh"]

vulnfeeds/cmd/cve-bulk-converter/Dockerfile → vulnfeeds/cmd/converters/cve/cve5/bulk-converter/Dockerfile

@@ -23,13 +23,13 @@ RUN go mod download && go mod verify
 
 
 COPY ./ /src/
-RUN go build -o cve-bulk-converter ./cmd/cve-bulk-converter/
+RUN go build -o cve-bulk-converter ./cmd/converters/cve/cve5/bulk-converter/
 
 FROM gcr.io/google.com/cloudsdktool/google-cloud-cli:alpine@sha256:feca5d4cb9b422e124e6f28b8ed2e714160757eb383eaae712117c75f584aa2f
 RUN apk --no-cache add jq
 
 WORKDIR /root/
 COPY --from=go_build /src/cve-bulk-converter ./
-COPY ./cmd/cve-bulk-converter/run-cvelist-converter.sh ./
+COPY ./cmd/converters/cve/cve5/bulk-converter/run-cvelist-converter.sh ./
 
 ENTRYPOINT ["/root/run-cvelist-converter.sh"]

vulnfeeds/cmd/nvd-cve-osv/Dockerfile → vulnfeeds/cmd/converters/cve/nvd-cve-osv/Dockerfile

@@ -20,12 +20,12 @@ COPY go.mod go.sum ./
 RUN go mod download && go mod verify
 
 COPY . .
-RUN CGO_ENABLED=0 go build -v -o /usr/local/bin ./cmd/nvd-cve-osv ./cmd/download-cves
+RUN CGO_ENABLED=0 go build -v -o /usr/local/bin ./cmd/converters/cve/nvd-cve-osv ./cmd/mirrors/download-cves
 
 FROM gcr.io/google.com/cloudsdktool/google-cloud-cli:alpine@sha256:feca5d4cb9b422e124e6f28b8ed2e714160757eb383eaae712117c75f584aa2f
 RUN apk --no-cache add jq
 
 COPY --from=GO_BUILD /usr/local/bin/ ./usr/local/bin/
-COPY --from=GO_BUILD /go/src/cmd/nvd-cve-osv/run_cve_to_osv_generation.sh ./usr/local/bin/
+COPY --from=GO_BUILD /go/src/cmd/converters/cve/nvd-cve-osv/run_cve_to_osv_generation.sh ./usr/local/bin/
 
 CMD ["/usr/local/bin/run_cve_to_osv_generation.sh"]

vulnfeeds/cmd/nvd-cve-osv/build.sh → vulnfeeds/cmd/converters/cve/nvd-cve-osv/build.sh

@@ -21,5 +21,5 @@ cd ../../
 
 docker build \
   -t gcr.io/oss-vdb/nvd-cve-osv:latest \
-  -f cmd/nvd-cve-osv/Dockerfile --pull . && \
+  -f cmd/cve/nvd-cve-osv/Dockerfile --pull . && \
 gcloud docker -- push gcr.io/oss-vdb/nvd-cve-osv:latest

vulnfeeds/cmd/debian/Dockerfile → vulnfeeds/cmd/converters/debian/Dockerfile

@@ -22,14 +22,14 @@ COPY ./go.sum /src/go.sum
 RUN go mod download
 
 COPY ./ /src/
-RUN go build -o debian ./cmd/debian/
+RUN go build -o debian ./cmd/converters/debian/
 
 
 FROM gcr.io/google.com/cloudsdktool/google-cloud-cli:alpine@sha256:feca5d4cb9b422e124e6f28b8ed2e714160757eb383eaae712117c75f584aa2f
 
 WORKDIR /root/
 COPY --from=GO_BUILD /src/debian ./
-COPY ./cmd/debian/run_debian_convert.sh ./
+COPY ./cmd/converters/debian/run_debian_convert.sh ./
 
 RUN chmod 755 ./run_debian_convert.sh
 

vulnfeeds/cmd/debian/main_test.go → vulnfeeds/cmd/converters/debian/main_test.go

@@ -40,7 +40,7 @@ func sortAffected(affected []*osvschema.Affected) {
 
 func loadTestData(t *testing.T, cveName string) cves.Vulnerability {
 	t.Helper()
-	fileName := fmt.Sprintf("../../test_data/nvdcve-2.0/%s.json", cveName)
+	fileName := fmt.Sprintf("../../../test_data/nvdcve-2.0/%s.json", cveName)
 	file, err := os.Open(fileName)
 	if err != nil {
 		t.Fatalf("Failed to load test data from %q: %#v", fileName, err)
@@ -65,7 +65,7 @@ func TestGenerateOSVFromDebianTracker(t *testing.T) {
 	now := time.Date(2024, 7, 1, 0, 0, 0, 0, time.UTC)
 
 	var trackerData DebianSecurityTrackerData
-	if err := json.Unmarshal(mustRead(t, "../../test_data/debian/debian_security_tracker_mock.json"), &trackerData); err != nil {
+	if err := json.Unmarshal(mustRead(t, "../../../test_data/debian/debian_security_tracker_mock.json"), &trackerData); err != nil {
 		t.Fatalf("Failed to unmarshal test data: %v", err)
 	}
 

vulnfeeds/cmd/cpe-repo-gen/Dockerfile → vulnfeeds/cmd/mirrors/cpe-repo-gen/Dockerfile

@@ -22,13 +22,13 @@ COPY ./go.sum /src/go.sum
 RUN go mod download
 
 COPY ./ /src/
-RUN CGO_ENABLED=0 go build -o cpe-repo-gen ./cmd/cpe-repo-gen
+RUN CGO_ENABLED=0 go build -o cpe-repo-gen ./cmd/mirrors/cpe-repo-gen
 
 FROM gcr.io/google.com/cloudsdktool/google-cloud-cli:alpine@sha256:feca5d4cb9b422e124e6f28b8ed2e714160757eb383eaae712117c75f584aa2f
 
 RUN apk add --no-cache unzip
 
 COPY --from=GO_BUILD /src/cpe-repo-gen ./
-COPY ./cmd/cpe-repo-gen/cpe-repo-gen_map.sh ./
+COPY ./cmd/mirrors/cpe-repo-gen/cpe-repo-gen_map.sh ./
 
 ENTRYPOINT ["/cpe-repo-gen_map.sh"]

vulnfeeds/cmd/download-cves/Dockerfile → vulnfeeds/cmd/mirrors/download-cves/Dockerfile

@@ -22,12 +22,12 @@ COPY ./go.sum /src/go.sum
 RUN go mod download
 
 COPY ./ /src/
-RUN go build -o download-cves ./cmd/download-cves/
+RUN go build -o download-cves ./cmd/mirrors/download-cves/
 
 FROM gcr.io/google.com/cloudsdktool/google-cloud-cli:alpine@sha256:feca5d4cb9b422e124e6f28b8ed2e714160757eb383eaae712117c75f584aa2f
 
 WORKDIR /usr/local/bin
 COPY --from=GO_BUILD /src/download-cves ./
-COPY ./cmd/download-cves/mirror_nvd.sh ./
+COPY ./cmd/mirrors/download-cves/mirror_nvd.sh ./
 
 ENTRYPOINT ["/usr/local/bin/mirror_nvd.sh"]

vulnfeeds/cves/versions.go

@@ -836,7 +836,7 @@ func RemoveQuoting(s string) (result string) {
 }
 
 // Parse a well-formed CPE string into a struct.
-func ParseCPE(formattedString string) (*models.CPE, error) {
+func ParseCPE(formattedString string) (*models.CPEString, error) {
 	if !strings.HasPrefix(formattedString, "cpe:") {
 		return nil, fmt.Errorf("%q does not have expected 'cpe:' prefix", formattedString)
 	}
@@ -847,7 +847,7 @@ func ParseCPE(formattedString string) (*models.CPE, error) {
 		return nil, err
 	}
 
-	return &models.CPE{
+	return &models.CPEString{
 		CPEVersion: strings.Split(formattedString, ":")[1],
 		Part:       wfn.GetString("part"),
 		Vendor:     RemoveQuoting(wfn.GetString("vendor")),

vulnfeeds/cves/versions_test.go

@@ -43,7 +43,7 @@ func TestParseCPE(t *testing.T) {
 	tests := []struct {
 		description       string
 		inputCPEString    string
-		expectedCPEStruct *models.CPE
+		expectedCPEStruct *models.CPEString
 		expectedOk        bool
 	}{
 		{
@@ -67,7 +67,7 @@ func TestParseCPE(t *testing.T) {
 		},
 		{
 			description: "valid input (hardware)", inputCPEString: "cpe:2.3:h:intel:core_i3-1005g1:-:*:*:*:*:*:*:*",
-			expectedCPEStruct: &models.CPE{
+			expectedCPEStruct: &models.CPEString{
 				CPEVersion: "2.3",
 				Part:       "h",
 				Vendor:     "intel",
@@ -86,7 +86,7 @@ func TestParseCPE(t *testing.T) {
 		{
 			description:    "valid input (software)",
 			inputCPEString: "cpe:2.3:a:gitlab:gitlab:*:*:*:*:community:*:*:*",
-			expectedCPEStruct: &models.CPE{
+			expectedCPEStruct: &models.CPEString{
 				CPEVersion: "2.3",
 				Part:       "a",
 				Vendor:     "gitlab",
@@ -105,7 +105,7 @@ func TestParseCPE(t *testing.T) {
 		{
 			description:    "valid input (software) with embedded colons",
 			inputCPEString: "cpe:2.3:a:http\\:\\:daemon_project:http\\:\\:daemon:*:*:*:*:*:*:*:*",
-			expectedCPEStruct: &models.CPE{
+			expectedCPEStruct: &models.CPEString{
 				CPEVersion: "2.3",
 				Part:       "a",
 				Vendor:     "http::daemon_project",
@@ -124,7 +124,7 @@ func TestParseCPE(t *testing.T) {
 		{
 			description:    "valid input (software) with escaped characters",
 			inputCPEString: "cpe:2.3:a:bloodshed:dev-c\\+\\+:4.9.9.2:*:*:*:*:*:*:*",
-			expectedCPEStruct: &models.CPE{
+			expectedCPEStruct: &models.CPEString{
 				CPEVersion: "2.3",
 				Part:       "a",
 				Vendor:     "bloodshed",

vulnfeeds/models/types.go

@@ -219,7 +219,7 @@ func (vi *VersionInfo) Duplicated(candidate AffectedCommit) bool {
 	return false
 }
 
-type CPE struct {
+type CPEString struct {
 	CPEVersion string
 	Part       string
 	Vendor     string

vulnfeeds/vulns/vulns.go

@@ -81,7 +81,6 @@ const (
 	Spaces                             // Contains space characters
 	Empty                              // Contains no entry
 	Filler                             // Has been determined to be a filler word
-
 )
 
 // AttachExtractedVersionInfo converts the models.VersionInfo struct to OSV GIT and ECOSYSTEM AffectedRanges and AffectedPackage.