From ff0e2873676d7c77d8252c197175b4e6415c4454 Mon Sep 17 00:00:00 2001 From: varshith Date: Wed, 17 Apr 2024 15:34:31 +0200 Subject: [PATCH 1/7] container workdir removed --- Dockerfile | 1 - 1 file changed, 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 4dc5e55..0419f24 100644 --- a/Dockerfile +++ b/Dockerfile @@ -7,7 +7,6 @@ ENV CGO_ENABLED=0 RUN go build -o app . FROM alpine:3.17 -WORKDIR /gen_app RUN apk add --no-cache --repository=https://dl-cdn.alpinelinux.org/alpine/edge/community hugo COPY --from=build /lp_app/app . COPY --from=build /lp_app/web web/ From c872d78dc2227ede850bb0c0376e6ad8bfb2b1a8 Mon Sep 17 00:00:00 2001 From: varshith Date: Wed, 17 Apr 2024 15:36:02 +0200 Subject: [PATCH 2/7] improved logging and error handling --- download_metadata.go | 6 ++---- main.go | 1 + markdown_writer.go | 15 ++++++++------- static_files_uploader.go | 2 +- 4 files changed, 12 insertions(+), 12 deletions(-) diff --git a/download_metadata.go b/download_metadata.go index 3d8a77c..374b848 100644 --- a/download_metadata.go +++ b/download_metadata.go @@ -29,7 +29,7 @@ func metadataDownloader(Metadataclient *MetadataBackend) { for paginator.HasMorePages() { page, err := paginator.NextPage(context.TODO()) if err != nil { - log.Fatalln("error:", err) + log.Fatalln("Error while paginating the s3 bucket", err) } for _, obj := range page.Contents { err := downloadToFile(manager, LocalDirectory, Bucket, aws.ToString(obj.Key)) @@ -58,12 +58,10 @@ func downloadToFile(downloader *manager.Downloader, targetDirectory, bucket, key log.Fatal("Error while writing XML files to folder", err) } defer fd.Close() - // Download the file using the AWS SDK for Go - //fmt.Printf("Downloading s3://%s/%s to %s...\n", bucket, key, file) _, err = downloader.Download(context.TODO(), fd, &s3.GetObjectInput{Bucket: &bucket, Key: &key}) if err != nil { - log.Infoln("Failed to download metadatafiles") + log.Fatal("Failed to download metadatafiles", err) } return err } diff --git a/main.go b/main.go index f0d3a2d..8301cd3 100644 --- a/main.go +++ b/main.go @@ -7,6 +7,7 @@ import ( ) func main() { + log.SetLevel(log.InfoLevel) log.Infoln("started app successfully") mConf := getMetadataConfig() Metadataclient := connectMetadatas3(mConf) diff --git a/markdown_writer.go b/markdown_writer.go index e08939b..c6676dd 100644 --- a/markdown_writer.go +++ b/markdown_writer.go @@ -5,6 +5,8 @@ import ( "os" "path/filepath" "strings" + + log "github.com/sirupsen/logrus" ) func markDownCreator() { @@ -16,14 +18,14 @@ func markDownCreator() { // Create the Markdown directory if it doesn't exist if err := os.MkdirAll(markdownDir, 0755); err != nil { - fmt.Println("Error creating directory:", err) + log.Fatal("Error creating directory:", err) os.Exit(1) } // Walk through the XML directory err := filepath.Walk(xmlDirPath, func(xmlFilePath string, info os.FileInfo, err error) error { if err != nil { - fmt.Printf("Error accessing file %s: %v\n", xmlFilePath, err) + log.Fatal("Error accessing file", err) return nil } // Skip directories @@ -52,7 +54,7 @@ Filename of the associated XML file: %s mdFileName := filepath.Join(markdownDir, fileNameWithoutExt+".md") mdFile, err := os.Create(mdFileName) if err != nil { - fmt.Printf("Error creating Markdown file %s: %v\n", mdFileName, err) + log.Fatal("Error creating Markdown file %V", err) return nil } defer mdFile.Close() @@ -60,17 +62,16 @@ Filename of the associated XML file: %s // Write Markdown content to the file _, err = mdFile.WriteString(markdownContent) if err != nil { - fmt.Printf("Error writing to Markdown file %s: %v\n", mdFileName, err) + log.Fatal("Error writing to Markdown file %V", mdFileName, err) return nil } - fmt.Printf("Markdown file %s created successfully!\n", mdFileName) + log.Infoln("Markdown file %S created successfully!\n", mdFileName) return nil }) if err != nil { - fmt.Println("Error walking through directory:", err) - os.Exit(1) + log.Fatal("Error walking through directory:", err) } } diff --git a/static_files_uploader.go b/static_files_uploader.go index 80f562f..0369b63 100644 --- a/static_files_uploader.go +++ b/static_files_uploader.go @@ -63,7 +63,7 @@ func test(DeploymenClient *DeploymentBackend) { Body: bytes.NewReader(buf), ContentType: aws.String(contentType), }) - println(contentType) + log.Debug(contentType) file.Close() if err != nil { log.Fatalln("Failed to upload", path, err) From f32f5fb33d6d36d43b7637a291835692f9b91c2f Mon Sep 17 00:00:00 2001 From: varshith Date: Wed, 17 Apr 2024 15:36:33 +0200 Subject: [PATCH 3/7] abort if no metadata files found --- s3.go | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/s3.go b/s3.go index dad6aa1..e45fb28 100644 --- a/s3.go +++ b/s3.go @@ -43,7 +43,7 @@ func connectMetadatas3(mConf MetadataS3Config) *MetadataBackend { Client: client, Bucket: mConf.Bucket, } - _, err = metadata_client.Client.ListObjectsV2(context.TODO(), &s3.ListObjectsV2Input{ + resp, err := metadata_client.Client.ListObjectsV2(context.TODO(), &s3.ListObjectsV2Input{ Bucket: aws.String(metadata_client.Bucket), }) if err != nil { @@ -51,6 +51,17 @@ func connectMetadatas3(mConf MetadataS3Config) *MetadataBackend { } else { log.Infoln("Connection established to metadata bucket", metadata_client.Bucket) } + + // Abort if 0 metadata files found in bucket + numberOfFiles := 0 + for _, obj := range resp.Contents { + if obj.Key != nil { + numberOfFiles++ + } + } + if numberOfFiles == 0 { + log.Fatal("No Metadata files found in bucket. Length of files ", numberOfFiles) + } return metadata_client } From 1d01936d33af0b4e7797961024616e20ce58a3d1 Mon Sep 17 00:00:00 2001 From: varshith Date: Wed, 17 Apr 2024 15:37:56 +0200 Subject: [PATCH 4/7] read only root file error and chart version bump --- charts/landing-pages-chart/Chart.yaml | 2 +- charts/landing-pages-chart/templates/lp_cj.yaml | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/charts/landing-pages-chart/Chart.yaml b/charts/landing-pages-chart/Chart.yaml index 0946b84..6e46e6f 100644 --- a/charts/landing-pages-chart/Chart.yaml +++ b/charts/landing-pages-chart/Chart.yaml @@ -15,7 +15,7 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.1.4 +version: 0.1.5 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to diff --git a/charts/landing-pages-chart/templates/lp_cj.yaml b/charts/landing-pages-chart/templates/lp_cj.yaml index aced24e..61ab252 100644 --- a/charts/landing-pages-chart/templates/lp_cj.yaml +++ b/charts/landing-pages-chart/templates/lp_cj.yaml @@ -31,6 +31,7 @@ spec: - name: lp-generator-container image: {{ .Values.images.lp_image }} imagePullPolicy: Always + command: ["sh" , "-c", "cp app -r web tmp/ && /tmp/app"] env: - name: CONFIGFILE value: "/.secrets/config.yaml" From ecf4f59f1c93f6463015064e49101f43f31570b1 Mon Sep 17 00:00:00 2001 From: varshith Date: Thu, 25 Apr 2024 13:42:38 +0200 Subject: [PATCH 5/7] data fetch short code refinement --- web/layouts/shortcodes/datafetch.html | 46 +++++++++++++++++++++++++-- 1 file changed, 43 insertions(+), 3 deletions(-) diff --git a/web/layouts/shortcodes/datafetch.html b/web/layouts/shortcodes/datafetch.html index d25fc6b..00bd916 100644 --- a/web/layouts/shortcodes/datafetch.html +++ b/web/layouts/shortcodes/datafetch.html @@ -1,11 +1,51 @@ - {{- $variable := .Get "variable" -}}
{{ $data := index .Site.Data.datasets $variable }} {{ if $data }} - {{ range $data.LANDING_PAGE.ATTRIBUTES.STRING_ATTRIBUTE }} -

{{ .TAG }} : {{ .VALUE }}

+ + + {{ $stringAttributes := index $data.LANDING_PAGE.ATTRIBUTES.STRING_ATTRIBUTE }} + {{ $filtered := where $stringAttributes "TAG" "!=" "header" }} + {{ range $filtered }} + + + + + + {{ end }} + + {{ $setAttributes := index $data.LANDING_PAGE.ATTRIBUTES.SET_ATTRIBUTE }} + {{ range $setAttributes }} + + {{ $keywords := slice }} + {{ if eq .TAG "anatomical_sites"}} + + + {{ range .VALUE.STRING_ATTRIBUTE }} + {{ $keywords = $keywords | append .VALUE }} + {{ end }} + + + {{ else }} + + + {{ range .VALUE.STRING_ATTRIBUTE }} + + {{ end }} + + {{ end }} + + {{ $numAttributes := index $data.LANDING_PAGE.ATTRIBUTES.NUMERIC_ATTRIBUTE }} + {{ range $numAttributes }} + + + + + {{ end }} +
{{ replaceRE `(_{1,})` " " .TAG }}{{ .VALUE }}
{{ replaceRE `(_{1,})` " " .TAG }}{{ delimit $keywords ", " }}
{{ replaceRE `(_{1,})` " " .TAG }}{{ . }} + {{ end }} +
{{ replaceRE `(_{1,})` " " .TAG }}{{ .VALUE }}
{{ else }}

Data for '{{ $variable }}' not found.

{{ end }} From 21b6532100df7bf07d259ff88ddf1bd671a390cc Mon Sep 17 00:00:00 2001 From: varshith Date: Mon, 6 May 2024 13:12:57 +0200 Subject: [PATCH 6/7] web changes for rems demo --- .gitignore | 2 +- web/layouts/datasets/list.html | 6 ++-- web/layouts/shortcodes/datafetch.html | 40 ++++++++++++++------------- 3 files changed, 25 insertions(+), 23 deletions(-) diff --git a/.gitignore b/.gitignore index 8c9ee7f..49e38a1 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,4 @@ web/public/* -web/content/datasets/test* +web/content/datasets/test* web/data/* dev_utils/config.yaml \ No newline at end of file diff --git a/web/layouts/datasets/list.html b/web/layouts/datasets/list.html index b68fce3..494b337 100644 --- a/web/layouts/datasets/list.html +++ b/web/layouts/datasets/list.html @@ -1,9 +1,9 @@ {{ define "main" }} +

Bigpicture Datasets

{{ .Params.datasets.name }} diff --git a/web/layouts/shortcodes/datafetch.html b/web/layouts/shortcodes/datafetch.html index 00bd916..e791a51 100644 --- a/web/layouts/shortcodes/datafetch.html +++ b/web/layouts/shortcodes/datafetch.html @@ -2,10 +2,16 @@
{{ $data := index .Site.Data.datasets $variable }} {{ if $data }} + {{ $stringAttributes2 := index $data.LANDING_PAGE.ATTRIBUTES.STRING_ATTRIBUTE }} + {{ $rte := where $stringAttributes2 "TAG" "header" }} + {{ range $rte }} +

{{ .VALUE }}

+ {{ end }} {{ $stringAttributes := index $data.LANDING_PAGE.ATTRIBUTES.STRING_ATTRIBUTE }} {{ $filtered := where $stringAttributes "TAG" "!=" "header" }} + {{ $filtered := where $filtered "TAG" "!=" "rems_link" }} {{ range $filtered }} @@ -14,29 +20,16 @@ {{ end }} - {{ $setAttributes := index $data.LANDING_PAGE.ATTRIBUTES.SET_ATTRIBUTE }} - {{ range $setAttributes }} - - {{ $keywords := slice }} - {{ if eq .TAG "anatomical_sites"}} + + {{ range index $data.LANDING_PAGE.ATTRIBUTES.SET_ATTRIBUTE }} - - {{ range .VALUE.STRING_ATTRIBUTE }} - {{ $keywords = $keywords | append .VALUE }} - {{ end }} - - - {{ else }} - - - {{ range .VALUE.STRING_ATTRIBUTE }} - + + {{ end }} - - {{ end }} + - {{ end }} {{ $numAttributes := index $data.LANDING_PAGE.ATTRIBUTES.NUMERIC_ATTRIBUTE }} {{ range $numAttributes }} @@ -49,4 +42,13 @@ {{ else }}

Data for '{{ $variable }}' not found.

{{ end }} + +

Comments

+ {{ $stringAttributes2 := index $data.LANDING_PAGE.ATTRIBUTES.STRING_ATTRIBUTE }} + {{ $rte := where $stringAttributes2 "TAG" "rems_link" }} + {{ range $rte }} +

Apply for access

 +

If you are interested to access data, apply through clicking link above

+ + {{ end }} From 0118ccc6964f5c14f07f955b743f58841c383547 Mon Sep 17 00:00:00 2001 From: varshith Date: Mon, 6 May 2024 13:13:46 +0200 Subject: [PATCH 7/7] domain name change --- web/config.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/web/config.toml b/web/config.toml index 0cef330..8e311c3 100644 --- a/web/config.toml +++ b/web/config.toml @@ -1,4 +1,4 @@ -baseurl = "https://lp.bp.nbis.se" +baseurl = "https://datasets.bp.nbis.se" title = "Bigpicture datasets" languageCode = "en-us"
{{ replaceRE `(_{1,})` " " .TAG }}
{{ replaceRE `(_{1,})` " " .TAG }}{{ delimit $keywords ", " }}
{{ replaceRE `(_{1,})` " " .TAG }}{{ . }} + {{ replaceRE `(_{1,})` " " .TAG }}{{ .VALUE.STRING_ATTRIBUTE.VALUE }}