cmd/modinfo: a tool for getting module info

Give this command-line tool a module path and it will print the number
of importers for each package in the module.

Change-Id: Ib93eb6bec1b7f049bda43c5f7f1b05a1281fb8e0
Reviewed-on: https://go-review.googlesource.com/c/vulndb/+/513499
Run-TryBot: Jonathan Amsterdam <[email protected]>
Reviewed-by: Tatiana Bradley <[email protected]>
TryBot-Result: Gopher Robot <[email protected]>

cmd/modinfo/internal/pkgsitedb/db.go

@@ -0,0 +1,112 @@
+// Copyright 2022 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+//go:build !plan9
+
+// Package pkgsitedb provides functionality for connecting to the pkgsite
+// database.
+package pkgsitedb
+
+import (
+	"context"
+	"database/sql"
+	"fmt"
+	"regexp"
+
+	secretmanager "cloud.google.com/go/secretmanager/apiv1"
+	smpb "cloud.google.com/go/secretmanager/apiv1/secretmanagerpb"
+	_ "github.com/lib/pq"
+)
+
+type Config struct {
+	User           string
+	PasswordSecret string
+	Password       string
+	Host           string
+	Port           string
+	DBName         string
+}
+
+// Open creates a connection to the pkgsite database.
+func Open(ctx context.Context, cfg Config) (_ *sql.DB, err error) {
+	if cfg.Password == "" {
+		var err error
+		cfg.Password, err = getSecret(ctx, cfg.PasswordSecret)
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	connString := fmt.Sprintf(
+		"user='%s' password='%s' host='%s' port=%s dbname='%s' sslmode='disable'",
+		cfg.User, cfg.Password, cfg.Host, cfg.Port, cfg.DBName)
+	db, err := sql.Open("postgres", connString)
+	if err != nil {
+		return nil, err
+	}
+	if err := db.PingContext(ctx); err != nil {
+		return nil, err
+	}
+	return db, nil
+}
+
+type Module struct {
+	Path     string
+	Packages []*Package
+}
+
+type Package struct {
+	Path         string
+	Version      string
+	NumImporters int
+}
+
+func QueryModule(ctx context.Context, db *sql.DB, modulePath string) (*Module, error) {
+	query := `
+		SELECT package_path, version, imported_by_count
+		FROM search_documents
+		WHERE module_path = $1
+		ORDER BY 3 DESC
+	`
+	rows, err := db.QueryContext(ctx, query, modulePath)
+	if err != nil {
+		return nil, err
+	}
+	defer rows.Close()
+	m := &Module{Path: modulePath}
+	for rows.Next() {
+		var p Package
+		if err := rows.Scan(&p.Path, &p.Version, &p.NumImporters); err != nil {
+			return nil, err
+		}
+		m.Packages = append(m.Packages, &p)
+	}
+	if err := rows.Err(); err != nil {
+		return nil, err
+	}
+	return m, nil
+}
+
+var passwordRegexp = regexp.MustCompile(`password=\S+`)
+
+func redactPassword(dbinfo string) string {
+	return passwordRegexp.ReplaceAllLiteralString(dbinfo, "password=REDACTED")
+}
+
+// getSecret retrieves a secret from the GCP Secret Manager.
+// secretFullName should be of the form "projects/PROJECT/secrets/NAME".
+func getSecret(ctx context.Context, secretFullName string) (_ string, err error) {
+	client, err := secretmanager.NewClient(ctx)
+	if err != nil {
+		return "", err
+	}
+	defer client.Close()
+	result, err := client.AccessSecretVersion(ctx, &smpb.AccessSecretVersionRequest{
+		Name: secretFullName + "/versions/latest",
+	})
+	if err != nil {
+		return "", err
+	}
+	return string(result.Payload.Data), nil
+}

cmd/modinfo/internal/pkgsitedb/db_plan9.go

@@ -0,0 +1,26 @@
+// Copyright 2022 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+//go:build plan9
+
+package pkgsitedb
+
+import (
+	"context"
+	"database/sql"
+	"errors"
+
+	"golang.org/x/pkgsite-metrics/internal/config"
+	"golang.org/x/pkgsite-metrics/internal/scan"
+)
+
+var errDoesNotCompile = errors.New("github.com/lib/pq does not compile on plan9")
+
+func Open(ctx context.Context, cfg *config.Config) (_ *sql.DB, err error) {
+	return nil, errDoesNotCompile
+}
+
+func ModuleSpecs(ctx context.Context, db *sql.DB, minImportedByCount int) (specs []scan.ModuleSpec, err error) {
+	return nil, errDoesNotCompile
+}

cmd/modinfo/internal/pkgsitedb/db_test.go

@@ -0,0 +1,54 @@
+// Copyright 2022 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+//go:build !plan9
+
+package pkgsitedb
+
+import (
+	// imported to register the postgres database driver
+
+	"context"
+	"encoding/json"
+	"flag"
+	"fmt"
+	"os"
+	"testing"
+
+	_ "github.com/lib/pq"
+)
+
+var dbConfigFile = flag.String("dbconf", "", "filename with db config as JSON")
+
+func TestQueryModule(t *testing.T) {
+	if *dbConfigFile == "" {
+		t.Skip("missing -dbconf")
+	}
+	f, err := os.Open(*dbConfigFile)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer f.Close()
+	var cfg Config
+	if err := json.NewDecoder(f).Decode(&cfg); err != nil {
+		t.Fatal(err)
+	}
+	ctx := context.Background()
+
+	db, err := Open(ctx, cfg)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer db.Close()
+	if err := db.PingContext(ctx); err != nil {
+		t.Fatal(err)
+	}
+	m, err := QueryModule(ctx, db, "golang.org/x/tools")
+	if err != nil {
+		t.Fatal(err)
+	}
+	for _, p := range m.Packages {
+		fmt.Printf("%+v\n", p)
+	}
+}

cmd/modinfo/internal/pkgsitedb/secrets.go

@@ -0,0 +1,5 @@
+// Copyright 2023 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package pkgsitedb

cmd/modinfo/main.go

@@ -0,0 +1,95 @@
+// Copyright 2023 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Command modinfo displays module info from the pkgsite database.
+//
+// One-time setup: install the cloud sql proxy:
+//
+//	> curl -o cloud-sql-proxy https://storage.googleapis.com/cloud-sql-connectors/cloud-sql-proxy/v2.6.0/cloud-sql-proxy.linux.amd64
+//	> chmod +x cloud-sql-proxy
+//	> ./cloud-sql-proxy $MODINFO_DB?port=5429 &
+//
+// cloud-sql-proxy must be running in the background to run this command.
+// If it stops, restart it by running the last command above.
+//
+// You will also need to do
+//
+//	> gcloud auth application-default login
+//
+// if you haven't already.
+package main
+
+import (
+	"context"
+	"fmt"
+	"log"
+	"os"
+	"sort"
+	"strings"
+	"text/tabwriter"
+
+	"golang.org/x/exp/maps"
+	"golang.org/x/vulndb/cmd/modinfo/internal/pkgsitedb"
+)
+
+func main() {
+	args := os.Args[1:]
+	if len(args) == 0 {
+		log.Fatal("missing module paths")
+	}
+	var cfg pkgsitedb.Config
+	cfg.User = mustEnv("MODINFO_USER")
+	cfg.Password = mustEnv("MODINFO_PASSWORD")
+	cfg.Host = "127.0.0.1"
+	cfg.Port = "5429"
+	cfg.DBName = mustEnv("MODINFO_DBNAME")
+	ctx := context.Background()
+	db, err := pkgsitedb.Open(ctx, cfg)
+	if err != nil {
+		fmt.Fprintf(os.Stderr, `Could not open DB. You may need to run
+
+    cloud-sql-proxy $MODINFO_DB?port=5429 &
+
+Details: %v`, err)
+		fmt.Fprintln(os.Stderr)
+		os.Exit(1)
+	}
+	defer db.Close()
+	for _, modpath := range args {
+		mod, err := pkgsitedb.QueryModule(ctx, db, modpath)
+		if err != nil {
+			log.Fatal(err)
+		}
+		display(mod)
+	}
+}
+
+func mustEnv(ev string) string {
+	if r := os.Getenv(ev); r != "" {
+		return r
+	}
+	fmt.Fprintf(os.Stderr, "need value for environment variable %s\n", ev)
+	os.Exit(1)
+	return ""
+}
+
+func display(m *pkgsitedb.Module) {
+	if len(m.Packages) == 0 {
+		fmt.Printf("No packages for module %s; maybe it doesn't exist?\n", m.Path)
+		return
+	}
+	versionMap := map[string]bool{}
+	for _, p := range m.Packages {
+		versionMap[p.Version] = true
+	}
+	versions := maps.Keys(versionMap)
+	sort.Strings(versions)
+	fmt.Printf("==== %s @ %s ====\n", m.Path, strings.Join(versions, ", "))
+	tw := tabwriter.NewWriter(os.Stdout, 2, 4, 1, ' ', 0)
+	fmt.Fprintf(tw, "%s\t%s\n", "Import Path", "Importers")
+	for _, p := range m.Packages {
+		fmt.Fprintf(tw, "%s\t%5d\n", p.Path, p.NumImporters)
+	}
+	tw.Flush()
+}