diff --git a/.gitignore b/.gitignore index 8c9ee7f..49e38a1 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,4 @@ web/public/* -web/content/datasets/test* +web/content/datasets/test* web/data/* dev_utils/config.yaml \ No newline at end of file diff --git a/Dockerfile b/Dockerfile index 4dc5e55..0419f24 100644 --- a/Dockerfile +++ b/Dockerfile @@ -7,7 +7,6 @@ ENV CGO_ENABLED=0 RUN go build -o app . FROM alpine:3.17 -WORKDIR /gen_app RUN apk add --no-cache --repository=https://dl-cdn.alpinelinux.org/alpine/edge/community hugo COPY --from=build /lp_app/app . COPY --from=build /lp_app/web web/ diff --git a/charts/landing-pages-chart/Chart.yaml b/charts/landing-pages-chart/Chart.yaml index 0946b84..6e46e6f 100644 --- a/charts/landing-pages-chart/Chart.yaml +++ b/charts/landing-pages-chart/Chart.yaml @@ -15,7 +15,7 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.1.4 +version: 0.1.5 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to diff --git a/charts/landing-pages-chart/templates/lp_cj.yaml b/charts/landing-pages-chart/templates/lp_cj.yaml index aced24e..61ab252 100644 --- a/charts/landing-pages-chart/templates/lp_cj.yaml +++ b/charts/landing-pages-chart/templates/lp_cj.yaml @@ -31,6 +31,7 @@ spec: - name: lp-generator-container image: {{ .Values.images.lp_image }} imagePullPolicy: Always + command: ["sh" , "-c", "cp app -r web tmp/ && /tmp/app"] env: - name: CONFIGFILE value: "/.secrets/config.yaml" diff --git a/download_metadata.go b/download_metadata.go index 3d8a77c..374b848 100644 --- a/download_metadata.go +++ b/download_metadata.go @@ -29,7 +29,7 @@ func metadataDownloader(Metadataclient *MetadataBackend) { for paginator.HasMorePages() { page, err := paginator.NextPage(context.TODO()) if err != nil { - log.Fatalln("error:", err) + log.Fatalln("Error while paginating the s3 bucket", err) } for _, obj := range page.Contents { err := downloadToFile(manager, LocalDirectory, Bucket, aws.ToString(obj.Key)) @@ -58,12 +58,10 @@ func downloadToFile(downloader *manager.Downloader, targetDirectory, bucket, key log.Fatal("Error while writing XML files to folder", err) } defer fd.Close() - // Download the file using the AWS SDK for Go - //fmt.Printf("Downloading s3://%s/%s to %s...\n", bucket, key, file) _, err = downloader.Download(context.TODO(), fd, &s3.GetObjectInput{Bucket: &bucket, Key: &key}) if err != nil { - log.Infoln("Failed to download metadatafiles") + log.Fatal("Failed to download metadatafiles", err) } return err } diff --git a/main.go b/main.go index f0d3a2d..8301cd3 100644 --- a/main.go +++ b/main.go @@ -7,6 +7,7 @@ import ( ) func main() { + log.SetLevel(log.InfoLevel) log.Infoln("started app successfully") mConf := getMetadataConfig() Metadataclient := connectMetadatas3(mConf) diff --git a/markdown_writer.go b/markdown_writer.go index e08939b..c6676dd 100644 --- a/markdown_writer.go +++ b/markdown_writer.go @@ -5,6 +5,8 @@ import ( "os" "path/filepath" "strings" + + log "github.com/sirupsen/logrus" ) func markDownCreator() { @@ -16,14 +18,14 @@ func markDownCreator() { // Create the Markdown directory if it doesn't exist if err := os.MkdirAll(markdownDir, 0755); err != nil { - fmt.Println("Error creating directory:", err) + log.Fatal("Error creating directory:", err) os.Exit(1) } // Walk through the XML directory err := filepath.Walk(xmlDirPath, func(xmlFilePath string, info os.FileInfo, err error) error { if err != nil { - fmt.Printf("Error accessing file %s: %v\n", xmlFilePath, err) + log.Fatal("Error accessing file", err) return nil } // Skip directories @@ -52,7 +54,7 @@ Filename of the associated XML file: %s mdFileName := filepath.Join(markdownDir, fileNameWithoutExt+".md") mdFile, err := os.Create(mdFileName) if err != nil { - fmt.Printf("Error creating Markdown file %s: %v\n", mdFileName, err) + log.Fatal("Error creating Markdown file %V", err) return nil } defer mdFile.Close() @@ -60,17 +62,16 @@ Filename of the associated XML file: %s // Write Markdown content to the file _, err = mdFile.WriteString(markdownContent) if err != nil { - fmt.Printf("Error writing to Markdown file %s: %v\n", mdFileName, err) + log.Fatal("Error writing to Markdown file %V", mdFileName, err) return nil } - fmt.Printf("Markdown file %s created successfully!\n", mdFileName) + log.Infoln("Markdown file %S created successfully!\n", mdFileName) return nil }) if err != nil { - fmt.Println("Error walking through directory:", err) - os.Exit(1) + log.Fatal("Error walking through directory:", err) } } diff --git a/s3.go b/s3.go index dad6aa1..e45fb28 100644 --- a/s3.go +++ b/s3.go @@ -43,7 +43,7 @@ func connectMetadatas3(mConf MetadataS3Config) *MetadataBackend { Client: client, Bucket: mConf.Bucket, } - _, err = metadata_client.Client.ListObjectsV2(context.TODO(), &s3.ListObjectsV2Input{ + resp, err := metadata_client.Client.ListObjectsV2(context.TODO(), &s3.ListObjectsV2Input{ Bucket: aws.String(metadata_client.Bucket), }) if err != nil { @@ -51,6 +51,17 @@ func connectMetadatas3(mConf MetadataS3Config) *MetadataBackend { } else { log.Infoln("Connection established to metadata bucket", metadata_client.Bucket) } + + // Abort if 0 metadata files found in bucket + numberOfFiles := 0 + for _, obj := range resp.Contents { + if obj.Key != nil { + numberOfFiles++ + } + } + if numberOfFiles == 0 { + log.Fatal("No Metadata files found in bucket. Length of files ", numberOfFiles) + } return metadata_client } diff --git a/static_files_uploader.go b/static_files_uploader.go index 80f562f..0369b63 100644 --- a/static_files_uploader.go +++ b/static_files_uploader.go @@ -63,7 +63,7 @@ func test(DeploymenClient *DeploymentBackend) { Body: bytes.NewReader(buf), ContentType: aws.String(contentType), }) - println(contentType) + log.Debug(contentType) file.Close() if err != nil { log.Fatalln("Failed to upload", path, err) diff --git a/web/config.toml b/web/config.toml index 0cef330..8e311c3 100644 --- a/web/config.toml +++ b/web/config.toml @@ -1,4 +1,4 @@ -baseurl = "https://lp.bp.nbis.se" +baseurl = "https://datasets.bp.nbis.se" title = "Bigpicture datasets" languageCode = "en-us" diff --git a/web/layouts/datasets/list.html b/web/layouts/datasets/list.html index b68fce3..494b337 100644 --- a/web/layouts/datasets/list.html +++ b/web/layouts/datasets/list.html @@ -1,9 +1,9 @@ {{ define "main" }} +

Bigpicture Datasets

{{ .Params.datasets.name }} diff --git a/web/layouts/shortcodes/datafetch.html b/web/layouts/shortcodes/datafetch.html index d25fc6b..e791a51 100644 --- a/web/layouts/shortcodes/datafetch.html +++ b/web/layouts/shortcodes/datafetch.html @@ -1,12 +1,54 @@ - {{- $variable := .Get "variable" -}}
{{ $data := index .Site.Data.datasets $variable }} {{ if $data }} - {{ range $data.LANDING_PAGE.ATTRIBUTES.STRING_ATTRIBUTE }} -

{{ .TAG }} : {{ .VALUE }}

+ {{ $stringAttributes2 := index $data.LANDING_PAGE.ATTRIBUTES.STRING_ATTRIBUTE }} + {{ $rte := where $stringAttributes2 "TAG" "header" }} + {{ range $rte }} +

{{ .VALUE }}

{{ end }} + + + {{ $stringAttributes := index $data.LANDING_PAGE.ATTRIBUTES.STRING_ATTRIBUTE }} + {{ $filtered := where $stringAttributes "TAG" "!=" "header" }} + {{ $filtered := where $filtered "TAG" "!=" "rems_link" }} + {{ range $filtered }} + + + + + + {{ end }} + + + {{ range index $data.LANDING_PAGE.ATTRIBUTES.SET_ATTRIBUTE }} + + + + + {{ end }} + + + + + {{ $numAttributes := index $data.LANDING_PAGE.ATTRIBUTES.NUMERIC_ATTRIBUTE }} + {{ range $numAttributes }} + + + + + {{ end }} +
{{ replaceRE `(_{1,})` " " .TAG }}{{ .VALUE }}
{{ replaceRE `(_{1,})` " " .TAG }}{{ .VALUE.STRING_ATTRIBUTE.VALUE }}
{{ replaceRE `(_{1,})` " " .TAG }}{{ .VALUE }}
{{ else }}

Data for '{{ $variable }}' not found.

{{ end }} + +

Comments

+ {{ $stringAttributes2 := index $data.LANDING_PAGE.ATTRIBUTES.STRING_ATTRIBUTE }} + {{ $rte := where $stringAttributes2 "TAG" "rems_link" }} + {{ range $rte }} +

Apply for access

 +

If you are interested to access data, apply through clicking link above

+ + {{ end }}