From d7811d067dc673682d91cdb09a974b127eb501b6 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Mon, 6 Nov 2023 21:04:30 +0100 Subject: [PATCH 01/33] fix(deps): update module github.com/tidwall/gjson to v1.17.0 (#1476) Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- go.mod | 2 +- go.sum | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/go.mod b/go.mod index b9420e1d0..625b266e7 100644 --- a/go.mod +++ b/go.mod @@ -52,7 +52,7 @@ require ( github.com/sirupsen/logrus v1.9.3 github.com/skratchdot/open-golang v0.0.0-20200116055534-eef842397966 github.com/thoas/go-funk v0.9.3 - github.com/tidwall/gjson v1.16.0 + github.com/tidwall/gjson v1.17.0 github.com/x-cray/logrus-prefixed-formatter v0.5.2 github.com/xo/dburl v0.16.0 golang.org/x/crypto v0.14.0 diff --git a/go.sum b/go.sum index a8feb52c1..daa2409bc 100644 --- a/go.sum +++ b/go.sum @@ -347,6 +347,8 @@ github.com/thoas/go-funk v0.9.3 h1:7+nAEx3kn5ZJcnDm2Bh23N2yOtweO14bi//dvRtgLpw= github.com/thoas/go-funk v0.9.3/go.mod h1:+IWnUfUmFO1+WVYQWQtIJHeRRdaIyyYglZN7xzUPe4Q= github.com/tidwall/gjson v1.16.0 h1:SyXa+dsSPpUlcwEDuKuEBJEz5vzTvOea+9rjyYodQFg= github.com/tidwall/gjson v1.16.0/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= +github.com/tidwall/gjson v1.17.0 h1:/Jocvlh98kcTfpN2+JzGQWQcqrPQwDrVEMApx/M5ZwM= +github.com/tidwall/gjson v1.17.0/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA= github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM= github.com/tidwall/pretty v1.2.0 h1:RWIZEg2iJ8/g6fDDYzMpobmaoGh5OLl4AXtGUGPcqCs= From 773ebc3d0f69960e679c4d83eed809d3cdeebe9a Mon Sep 17 00:00:00 2001 From: toshski <104477758+toshski@users.noreply.github.com> Date: Tue, 7 Nov 2023 09:04:50 +1300 Subject: [PATCH 02/33] scraper: Add Actor Scraping for PornCornVR (#1478) * Add Actor Scraping for PornCornVR * Typos and grammat --- pkg/models/model_external_reference.go | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/pkg/models/model_external_reference.go b/pkg/models/model_external_reference.go index a589c9540..f51ff058d 100644 --- a/pkg/models/model_external_reference.go +++ b/pkg/models/model_external_reference.go @@ -859,6 +859,10 @@ func (scrapeRules ActorScraperConfig) buildGenericActorScraperRules() { }) scrapeRules.GenericActorScrapingConfig["realjamvr scrape"] = siteDetails + // use the site rules just setup for realjamvr, just need to update the Domain to use + siteDetails.Domain = "porncornvr.com" + scrapeRules.GenericActorScrapingConfig["porncornvr scrape"] = siteDetails + siteDetails = GenericScraperRuleSet{} siteDetails.Domain = "povr.com" siteDetails.SiteRules = append(siteDetails.SiteRules, GenericActorScraperRule{XbvrField: "image_url", Selector: `script[type="application/ld+json"]`, PostProcessing: []PostProcessing{{Function: "jsonString", Params: []string{"image"}}}}) @@ -1045,6 +1049,7 @@ func (scrapeRules ActorScraperConfig) getSiteUrlMatchingRules() { var sites []Site + // if the scene_url in xbvr and stash typically matches, then no special rules required scrapeRules.StashSceneMatching["allvrporn-vrporn"] = StashSiteConfig{StashId: "44fd483b-85eb-4b22-b7f2-c92c1a50923a"} scrapeRules.StashSceneMatching["bvr"] = StashSiteConfig{StashId: "1ffbd972-7d69-4ccb-b7da-c6342a9c3d70"} scrapeRules.StashSceneMatching["cuties-vr"] = StashSiteConfig{StashId: "1e5240a8-29b3-41ed-ae28-fc9231eac449"} @@ -1077,8 +1082,11 @@ func (scrapeRules ActorScraperConfig) getSiteUrlMatchingRules() { scrapeRules.StashSceneMatching["vrsolos"] = StashSiteConfig{StashId: "b2d048da-9180-4e43-b41a-bdb4d265c8ec"} scrapeRules.StashSceneMatching["vrspy"] = StashSiteConfig{StashId: "513001ef-dff4-476d-840d-e22ef27e81ed"} scrapeRules.StashSceneMatching["wankitnowvr"] = StashSiteConfig{StashId: "acb1ed8f-4967-4c5a-b16a-7025bdeb75c5"} + scrapeRules.StashSceneMatching["porncornvr"] = StashSiteConfig{StashId: "9ecb1d29-64e8-4336-9bd2-5dda53341e29"} scrapeRules.StashSceneMatching["wetvr"] = StashSiteConfig{StashId: "981887d6-da48-4dfc-88d1-7ed13a2754f2"} + + // setup special rules to match scenes in xbvr and stashdb, rather than assuming scene_urls match scrapeRules.StashSceneMatching["wankzvr"] = StashSiteConfig{ StashId: "b04bca51-15ea-45ab-80f6-7b002fd4a02d", Rules: []SceneMatchRule{{XbvrField: "scene_id", XbvrMatch: `-\d+$`, XbvrMatchResultPosition: 0, StashRule: `(povr|wankzvr).com\/(.*)(-\d*?)\/?$`, StashMatchResultPosition: 3}}, From 7cdcfd322797a352d2e0cee2a8326e8c15db1077 Mon Sep 17 00:00:00 2001 From: toshski <104477758+toshski@users.noreply.github.com> Date: Tue, 7 Nov 2023 09:05:13 +1300 Subject: [PATCH 03/33] minor: Log Colly error massages (#1479) * Log Colly error maessages * Change Log Level to Error --- pkg/scrape/scrape.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/pkg/scrape/scrape.go b/pkg/scrape/scrape.go index ed2dbc07d..118556f6c 100644 --- a/pkg/scrape/scrape.go +++ b/pkg/scrape/scrape.go @@ -28,6 +28,11 @@ func createCollector(domains ...string) *colly.Collector { colly.UserAgent(UserAgent), ) + // Set error handler + c.OnError(func(r *colly.Response, err error) { + log.Errorf("Error visiting %s %s", r.Request.URL, err) + }) + c = createCallbacks(c) return c } From 6f6fb66d6b6a30f5661a95774ebd64fd3d4f6749 Mon Sep 17 00:00:00 2001 From: crwxaj <52156245+crwxaj@users.noreply.github.com> Date: Mon, 6 Nov 2023 21:05:31 +0100 Subject: [PATCH 04/33] minor: update to Go 1.20 (#1482) Co-authored-by: crwxaj --- .github/workflows/lint.yml | 2 +- .gitpod.dockerfile | 2 +- Makefile | 2 +- README.md | 2 +- go.mod | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 30cb22230..a45850704 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -15,7 +15,7 @@ jobs: - name: Install Go uses: actions/setup-go@v4 with: - go-version: 1.19.x + go-version: 1.20.x - name: Install Node.js uses: actions/setup-node@v3 diff --git a/.gitpod.dockerfile b/.gitpod.dockerfile index 1cb1bfe72..c7f31dc4c 100644 --- a/.gitpod.dockerfile +++ b/.gitpod.dockerfile @@ -4,7 +4,7 @@ ENV HOME=/home/gitpod WORKDIR $HOME USER gitpod -ENV GO_VERSION=1.19.6 \ +ENV GO_VERSION=1.20.10 \ GOPATH=$HOME/go-packages \ GOROOT=$HOME/go RUN export PATH=$(echo "$PATH" | sed -e 's|:/workspace/go/bin||' -e 's|:/home/gitpod/go/bin||' -e 's|:/home/gitpod/go-packages/bin||') diff --git a/Makefile b/Makefile index 315381520..43c32d03a 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -GORELEASER_CROSS_VERSION ?= v1.19.6 +GORELEASER_CROSS_VERSION ?= v1.20.10 SYSROOT_DIR ?= sysroots SYSROOT_ARCHIVE ?= sysroots.tar.bz2 diff --git a/README.md b/README.md index be878e7d6..7f856af64 100644 --- a/README.md +++ b/README.md @@ -73,7 +73,7 @@ Ask your questions and suggest features on [Discord](https://discord.gg/wdCHXAG) Make sure you have following installed: -- Go 1.19 +- Go 1.20 - Node.js 12.x - Yarn 1.17.x - air (run `go install github.com/cosmtrek/air@latest` outside project directory) diff --git a/go.mod b/go.mod index 625b266e7..464d5cb26 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module github.com/xbapps/xbvr -go 1.19 +go 1.20 require ( github.com/ProtonMail/go-appdir v1.1.0 From d2dc039174a95d13b1e4c6e5db0ecf54bf70bcc2 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Mon, 6 Nov 2023 21:06:17 +0100 Subject: [PATCH 05/33] fix(deps): update module github.com/xo/dburl to v0.17.0 (#1481) Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- go.mod | 2 +- go.sum | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/go.mod b/go.mod index 464d5cb26..33073b25f 100644 --- a/go.mod +++ b/go.mod @@ -54,7 +54,7 @@ require ( github.com/thoas/go-funk v0.9.3 github.com/tidwall/gjson v1.17.0 github.com/x-cray/logrus-prefixed-formatter v0.5.2 - github.com/xo/dburl v0.16.0 + github.com/xo/dburl v0.17.0 golang.org/x/crypto v0.14.0 golang.org/x/net v0.17.0 golang.org/x/oauth2 v0.11.0 diff --git a/go.sum b/go.sum index daa2409bc..eb561a58a 100644 --- a/go.sum +++ b/go.sum @@ -366,6 +366,8 @@ github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8 h1:nIPpBwaJSVYIxUFsDv3M8ofm github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8/go.mod h1:HUYIGzjTL3rfEspMxjDjgmT5uz5wzYJKVo23qUhYTos= github.com/xo/dburl v0.16.0 h1:jlBeGe8fnsW+vBYemte903WHQbJnZx7OpJZy2ofq+5g= github.com/xo/dburl v0.16.0/go.mod h1:B7/G9FGungw6ighV8xJNwWYQPMfn3gsi2sn5SE8Bzco= +github.com/xo/dburl v0.17.0 h1:sUgx24FNMbdxjutH6r4eg4y5S75RMjb/C+eGqnttcqU= +github.com/xo/dburl v0.17.0/go.mod h1:B7/G9FGungw6ighV8xJNwWYQPMfn3gsi2sn5SE8Bzco= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= go.etcd.io/bbolt v1.3.7 h1:j+zJOnnEjF/kyHlDDgGnVL/AIqIJPq8UoB2GSNfkUfQ= go.etcd.io/bbolt v1.3.7/go.mod h1:N9Mkw9X8x5fupy0IKsmuqVtoGDyxsaDlbk4Rd05IAQw= From 24325c7c65020d5ede79df3bce1fa0d7583b5c90 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Mon, 6 Nov 2023 21:06:50 +0100 Subject: [PATCH 06/33] fix(deps): update module github.com/rs/cors to v1.10.1 (#1475) Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- go.mod | 2 +- go.sum | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/go.mod b/go.mod index 33073b25f..ee96c0f11 100644 --- a/go.mod +++ b/go.mod @@ -48,7 +48,7 @@ require ( github.com/putdotio/go-putio v1.7.1 github.com/robertkrimen/otto v0.2.1 github.com/robfig/cron/v3 v3.0.1 - github.com/rs/cors v1.9.0 + github.com/rs/cors v1.10.1 github.com/sirupsen/logrus v1.9.3 github.com/skratchdot/open-golang v0.0.0-20200116055534-eef842397966 github.com/thoas/go-funk v0.9.3 diff --git a/go.sum b/go.sum index eb561a58a..d43c1671a 100644 --- a/go.sum +++ b/go.sum @@ -323,6 +323,8 @@ github.com/robfig/cron/v3 v3.0.1/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzG github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= github.com/rs/cors v1.9.0 h1:l9HGsTsHJcvW14Nk7J9KFz8bzeAWXn3CG6bgt7LsrAE= github.com/rs/cors v1.9.0/go.mod h1:XyqrcTp5zjWr1wsJ8PIRZssZ8b/WMcMf71DJnit4EMU= +github.com/rs/cors v1.10.1 h1:L0uuZVXIKlI1SShY2nhFfo44TYvDPQ1w4oFkUJNfhyo= +github.com/rs/cors v1.10.1/go.mod h1:XyqrcTp5zjWr1wsJ8PIRZssZ8b/WMcMf71DJnit4EMU= github.com/rwcarlsen/goexif v0.0.0-20190401172101-9e8deecbddbd h1:CmH9+J6ZSsIjUK3dcGsnCnO41eRBOnY12zwkn5qVwgc= github.com/rwcarlsen/goexif v0.0.0-20190401172101-9e8deecbddbd/go.mod h1:hPqNNc0+uJM6H+SuU8sEs5K5IQeKccPqeSjfgcKGgPk= github.com/saintfish/chardet v0.0.0-20120816061221-3af4cd4741ca h1:NugYot0LIVPxTvN8n+Kvkn6TrbMyxQiuvKdEwFdR9vI= From 9e442c1d35ec52af82c0db3272f383378ae099f8 Mon Sep 17 00:00:00 2001 From: crwxaj <52156245+crwxaj@users.noreply.github.com> Date: Mon, 6 Nov 2023 21:07:09 +0100 Subject: [PATCH 07/33] fix: fix Add Item button on empty filenames list, update scene details after edit (#1480) Co-authored-by: crwxaj --- ui/src/views/scenes/EditScene.vue | 27 +++++++-------------------- 1 file changed, 7 insertions(+), 20 deletions(-) diff --git a/ui/src/views/scenes/EditScene.vue b/ui/src/views/scenes/EditScene.vue index 9f317be15..a6b7cb69a 100644 --- a/ui/src/views/scenes/EditScene.vue +++ b/ui/src/views/scenes/EditScene.vue @@ -147,6 +147,9 @@ export default { } try { scene.files = JSON.parse(scene.filenames_arr) + if (scene.files == null) { + scene.files = [] + } } catch { scene.files = [] } @@ -207,27 +210,11 @@ export default { this.scene.filenames_arr = JSON.stringify(this.scene.files) this.scene.duration = String(this.scene.duration) // force to a string, if no change the UI sends an int, otherwise a string, nust be constant - ky.post(`/api/scene/edit/${this.scene.id}`, { json: { ...this.scene } }) - - this.scene.cast = this.scene.castArray.map(a => { - const find = this.scene.cast.find(o => o.name === a) - if (find) return find - return { - name: a, - count: 0 - } - }) - - this.scene.tags = this.scene.tagsArray.map(t => { - const find = this.scene.tags.find(o => o.name === t) - if (find) return find - return { - name: t, - count: 0 - } - }) + ky.post(`/api/scene/edit/${this.scene.id}`, { json: { ...this.scene } }).json().then(data => { + this.$store.commit('sceneList/updateScene', data) + this.$store.commit('overlay/showDetails', { scene: data }) + }) - this.$store.commit('sceneList/updateScene', this.scene) this.changesMade = false this.close() From e12f37a528f914f7b67f8bba1e18fe6eba83d6e5 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Mon, 6 Nov 2023 21:07:47 +0100 Subject: [PATCH 08/33] fix(deps): update module golang.org/x/oauth2 to v0.13.0 (#1483) Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- go.mod | 2 +- go.sum | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/go.mod b/go.mod index ee96c0f11..88a6f8b53 100644 --- a/go.mod +++ b/go.mod @@ -57,7 +57,7 @@ require ( github.com/xo/dburl v0.17.0 golang.org/x/crypto v0.14.0 golang.org/x/net v0.17.0 - golang.org/x/oauth2 v0.11.0 + golang.org/x/oauth2 v0.13.0 golang.org/x/sys v0.13.0 golang.org/x/text v0.13.0 gopkg.in/gormigrate.v1 v1.6.0 diff --git a/go.sum b/go.sum index d43c1671a..04000aaf4 100644 --- a/go.sum +++ b/go.sum @@ -420,6 +420,8 @@ golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAG golang.org/x/oauth2 v0.2.0/go.mod h1:Cwn6afJ8jrQwYMxQDTpISoXmXW9I6qF6vDeuuoX3Ibs= golang.org/x/oauth2 v0.11.0 h1:vPL4xzxBM4niKCW6g9whtaWVXTJf1U5e4aZxxFx/gbU= golang.org/x/oauth2 v0.11.0/go.mod h1:LdF7O/8bLR/qWK9DrpXmbHLTouvRHK0SgJl0GmDBchk= +golang.org/x/oauth2 v0.13.0 h1:jDDenyj+WgFtmV3zYVoi8aE2BwtXFLWOA67ZfNWftiY= +golang.org/x/oauth2 v0.13.0/go.mod h1:/JMhi4ZRXAf4HG9LiNmxvk+45+96RUlVThiH8FzNBn0= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= From 9a4b91dc44c93d33e895c41da7a342b8fc04cc59 Mon Sep 17 00:00:00 2001 From: crwxaj <52156245+crwxaj@users.noreply.github.com> Date: Mon, 13 Nov 2023 21:02:13 +0100 Subject: [PATCH 09/33] chore: code cleanup (staticcheck) (#1484) * code cleanup (staticcheck) * code cleanup * fix more checks, and disable some --------- Co-authored-by: crwxaj --- pkg/api/actors.go | 33 --------------------- pkg/api/akas.go | 2 +- pkg/api/api.go | 2 +- pkg/api/dms.go | 10 +------ pkg/api/files.go | 6 ++-- pkg/api/heresphere.go | 24 ++++------------ pkg/api/options.go | 5 ++-- pkg/api/scenes.go | 4 +-- pkg/api/trailers.go | 8 +++--- pkg/common/metrics.go | 2 +- pkg/config/scraper_list.go | 7 ++--- pkg/dms/dlna/dms/dms.go | 7 ++--- pkg/dms/staticcheck.conf | 2 ++ pkg/externalreference/log.go | 2 +- pkg/externalreference/stashdb.go | 19 ------------ pkg/models/db.go | 6 ++-- pkg/models/model_action.go | 3 +- pkg/models/model_action_actor.go | 3 +- pkg/models/model_actor.go | 8 ++---- pkg/models/model_aka.go | 5 ---- pkg/models/model_external_reference.go | 7 ++--- pkg/models/model_file.go | 3 +- pkg/models/model_history.go | 3 +- pkg/models/model_kv.go | 5 ++-- pkg/models/model_playlist.go | 3 +- pkg/models/model_scene.go | 16 +++++------ pkg/models/model_site.go | 5 ++-- pkg/models/model_tag.go | 3 +- pkg/models/model_taggroups.go | 5 ---- pkg/models/model_volume.go | 3 +- pkg/scrape/genericactorscraper.go | 40 +------------------------- pkg/scrape/javlibrary.go | 2 +- pkg/scrape/javutil.go | 12 ++++---- pkg/scrape/lethalhardcorevr.go | 15 ---------- pkg/scrape/realjamvr.go | 9 ++---- pkg/scrape/sexbabesvr.go | 8 ------ pkg/scrape/slrstudios.go | 2 +- pkg/scrape/stashdb.go | 12 ++------ pkg/scrape/vr3000.go | 3 +- pkg/server/heatmapproxy.go | 8 ++---- pkg/server/server.go | 4 +-- pkg/session/session.go | 9 +++--- pkg/tasks/content.go | 16 +++++------ pkg/tasks/deps.go | 2 +- pkg/tasks/heatmap.go | 3 +- pkg/tasks/log.go | 2 +- pkg/tasks/oshash.go | 4 +-- pkg/tray/main.go | 2 +- 48 files changed, 92 insertions(+), 272 deletions(-) create mode 100644 pkg/dms/staticcheck.conf diff --git a/pkg/api/actors.go b/pkg/api/actors.go index 0e808fdcc..f698f0a1b 100644 --- a/pkg/api/actors.go +++ b/pkg/api/actors.go @@ -487,39 +487,6 @@ func checkStringArrayChanged(field_name string, newValue *string, actorField *st } } -func checkURLArrayChanged(field_name string, newValue *string, actorField *string, actorId uint) { - if *actorField != *newValue { - var actorArray []models.ActorLink - var newArray []string - json.Unmarshal([]byte(*newValue), &newArray) - json.Unmarshal([]byte(*actorField), &actorArray) - for _, actorField := range actorArray { - exists := false - for _, newField := range newArray { - if newField == actorField.Url { - exists = true - } - } - if !exists { - models.AddActionActor(actorId, "edit_actor", "delete", field_name, actorField.Url) - } - } - for _, newField := range newArray { - exists := false - for _, actorField := range actorArray { - if newField == actorField.Url { - exists = true - } - } - if !exists { - models.AddActionActor(actorId, "edit_actor", "add", field_name, newField) - } - } - - *actorField = *newValue - } -} - func (i ActorResource) setActorImage(req *restful.Request, resp *restful.Response) { var r RequestSetActorImage err := req.ReadEntity(&r) diff --git a/pkg/api/akas.go b/pkg/api/akas.go index 7b9b7752d..734a3b330 100644 --- a/pkg/api/akas.go +++ b/pkg/api/akas.go @@ -207,7 +207,7 @@ func (i AkaResource) getAka(req *restful.Request, resp *restful.Response) { var aka models.Aka db, _ := models.GetDB() - err = aka.GetIfExistByPK(uint(sceneId)) + _ = aka.GetIfExistByPK(uint(sceneId)) db.Close() resp.WriteHeaderAndEntity(http.StatusOK, aka) diff --git a/pkg/api/api.go b/pkg/api/api.go index ac7cc99cb..883fa4935 100644 --- a/pkg/api/api.go +++ b/pkg/api/api.go @@ -5,7 +5,7 @@ import ( "github.com/xbapps/xbvr/pkg/common" ) -var log = common.Log +var log = &common.Log func APIError(req *restful.Request, resp *restful.Response, status int, err error) { resp.WriteError(status, err) diff --git a/pkg/api/dms.go b/pkg/api/dms.go index 995a0d29b..e1ddbabbf 100644 --- a/pkg/api/dms.go +++ b/pkg/api/dms.go @@ -6,7 +6,6 @@ import ( "net/http" "path/filepath" "strconv" - "time" restfulspec "github.com/emicklei/go-restful-openapi/v2" "github.com/emicklei/go-restful/v3" @@ -19,13 +18,6 @@ import ( type DMSResource struct{} -var ( - lastSessionID uint - lastSessionSceneID uint - lastSessionStart time.Time - lastSessionEnd time.Time -) - func (i DMSResource) WebService() *restful.WebService { tags := []string{"DMS"} @@ -112,6 +104,6 @@ func (i DMSResource) getFile(req *restful.Request, resp *restful.Response) { if err != nil { return } - http.Redirect(resp.ResponseWriter, req.Request, url, 302) + http.Redirect(resp.ResponseWriter, req.Request, url, http.StatusFound) } } diff --git a/pkg/api/files.go b/pkg/api/files.go index 4d41e7d33..12f39a877 100644 --- a/pkg/api/files.go +++ b/pkg/api/files.go @@ -331,7 +331,7 @@ func removeFileByFileId(fileId uint) models.Scene { if err == nil { deleted = true } else { - log.Errorf("Error deleting file ", err) + log.Errorf("error deleting file: %v", err) } case "putio": id, err := strconv.ParseInt(file.Path, 10, 64) @@ -343,7 +343,7 @@ func removeFileByFileId(fileId uint) models.Scene { if err == nil { deleted = true } else { - log.Errorf("Error deleting file ", err) + log.Errorf("error deleting file %v", err) } } @@ -355,7 +355,7 @@ func removeFileByFileId(fileId uint) models.Scene { } } } else { - log.Errorf("Error deleting file ", err) + log.Errorf("error deleting file %v", err) } return scene } diff --git a/pkg/api/heresphere.go b/pkg/api/heresphere.go index e6fcd5975..14e56efe0 100644 --- a/pkg/api/heresphere.go +++ b/pkg/api/heresphere.go @@ -4,9 +4,10 @@ import ( "encoding/base64" "encoding/json" "fmt" - "io/ioutil" + "io" "math" "net/http" + "os" "path/filepath" "sort" "strconv" @@ -107,7 +108,7 @@ type HereSphereAuthRequest struct { var RequestBody []byte func HeresphereAuthFilter(req *restful.Request, resp *restful.Response, chain *restful.FilterChain) { - RequestBody, _ = ioutil.ReadAll(req.Request.Body) + RequestBody, _ = io.ReadAll(req.Request.Body) if isDeoAuthEnabled() { var authorized bool var requestData HereSphereAuthRequest @@ -358,13 +359,7 @@ func (i HeresphereResource) getHeresphereScene(req *restful.Request, resp *restf if len(encoding.VideoSources) > 0 { hsp.Name = encoding.Name for _, source := range encoding.VideoSources { - hspSource := HeresphereSource{ - URL: source.URL, - Width: source.Width, - Height: source.Height, - Resolution: source.Resolution, - Size: source.Size, - } + hspSource := HeresphereSource(source) hsp.Sources = append(hsp.Sources, hspSource) } media = append(media, hsp) @@ -893,7 +888,7 @@ func ProcessHeresphereUpdates(scene *models.Scene, requestData HereSphereAuthReq } fName := filepath.Join(scene.Files[0].Path, strings.TrimSuffix(scene.Files[0].Filename, filepath.Ext(videoFile.Filename))+".hsp") - ioutil.WriteFile(fName, hspContent, 0644) + os.WriteFile(fName, hspContent, 0644) tasks.ScanLocalHspFile(fName, videoFile.VolumeID, scene.ID) } @@ -949,15 +944,6 @@ func findEndPos(requestData HereSphereAuthRequest) float64 { return endpos } -func matchCuepoint(findCuepoint models.SceneCuepoint, cuepointList []models.SceneCuepoint) int { - for idx, cuepoint := range cuepointList { - if cuepoint.Name == findCuepoint.Name && cuepoint.TimeStart == findCuepoint.TimeStart && *cuepoint.Track == *findCuepoint.Track && cuepoint.TimeEnd == findCuepoint.TimeEnd { - return idx - } - } - return -1 -} - func (i HeresphereResource) getHeresphereLibrary(req *restful.Request, resp *restful.Response) { log.Infof("getHeresphereLibrary called, enabled %v", config.Config.Interfaces.DeoVR.Enabled) if !config.Config.Interfaces.DeoVR.Enabled { diff --git a/pkg/api/options.go b/pkg/api/options.go index 52b1313fd..2a90981c8 100644 --- a/pkg/api/options.go +++ b/pkg/api/options.go @@ -5,7 +5,6 @@ import ( "crypto/sha1" "encoding/json" "fmt" - "io/ioutil" "net/http" "os" "path/filepath" @@ -71,7 +70,7 @@ type RequestSaveOptionsAdvanced struct { } type RequestSaveOptionsFunscripts struct { - ScrapeFunscripts bool `json:"scrapeFunscripts":` + ScrapeFunscripts bool `json:"scrapeFunscripts"` } type RequestSaveOptionsDLNA struct { Enabled bool `json:"enabled"` @@ -935,7 +934,7 @@ func (i ConfigResource) createCustomSite(req *restful.Request, resp *restful.Res scraperConfig.CustomScrapers.VrpornScrapers = scrapers["vrporn"] fName := filepath.Join(common.AppDir, "scrapers.json") list, _ := json.MarshalIndent(scraperConfig, "", " ") - ioutil.WriteFile(fName, list, 0644) + os.WriteFile(fName, list, 0644) resp.WriteHeader(http.StatusOK) } diff --git a/pkg/api/scenes.go b/pkg/api/scenes.go index 03ec3bff3..313077354 100644 --- a/pkg/api/scenes.go +++ b/pkg/api/scenes.go @@ -676,7 +676,7 @@ func (i SceneResource) deleteSceneCuepoint(req *restful.Request, resp *restful.R db.Delete(&cuepoint) var scene models.Scene - err = scene.GetIfExistByPK(uint(sceneId)) + _ = scene.GetIfExistByPK(uint(sceneId)) defer db.Close() resp.WriteHeaderAndEntity(http.StatusOK, scene) @@ -739,7 +739,7 @@ func (i SceneResource) selectScript(req *restful.Request, resp *restful.Response } } } - err = scene.GetIfExistByPK(uint(sceneId)) + _ = scene.GetIfExistByPK(uint(sceneId)) } db.Close() diff --git a/pkg/api/trailers.go b/pkg/api/trailers.go index 60fbab2fb..8794b1e08 100644 --- a/pkg/api/trailers.go +++ b/pkg/api/trailers.go @@ -3,7 +3,7 @@ package api import ( "encoding/json" "html" - "io/ioutil" + "io" "net/http" "regexp" "strings" @@ -21,7 +21,7 @@ func LoadHeresphereScene(url string) HeresphereVideo { return HeresphereVideo{} } - responseData, err := ioutil.ReadAll(response.Body) + responseData, err := io.ReadAll(response.Body) if err != nil { log.Errorf("Error from %s %s", url, err) } @@ -41,7 +41,7 @@ func LoadDeovrScene(url string) DeoScene { return DeoScene{} } - responseData, err := ioutil.ReadAll(response.Body) + responseData, err := io.ReadAll(response.Body) if err != nil { log.Errorf("Error from %s %s", url, err) } @@ -135,7 +135,7 @@ func LoadJson(scrapeParams string) models.VideoSourceResponse { return models.VideoSourceResponse{} } - responseData, err := ioutil.ReadAll(response.Body) + responseData, err := io.ReadAll(response.Body) if err != nil { log.Errorf("Error from %s %s", params.SceneUrl, err) } diff --git a/pkg/common/metrics.go b/pkg/common/metrics.go index 077c86e4c..f5e0658f7 100644 --- a/pkg/common/metrics.go +++ b/pkg/common/metrics.go @@ -9,7 +9,7 @@ import ( ) func GetMetric(name string) (*whisper.Whisper, error) { - retentions, err := whisper.ParseRetentionDefs("1m:1d,1h:60d,12h:20y") + retentions, _ := whisper.ParseRetentionDefs("1m:1d,1h:60d,12h:20y") path := filepath.Join(MetricsDir, name+".wsp") wsp, err := whisper.Create(path, retentions, whisper.Last, 0.5) diff --git a/pkg/config/scraper_list.go b/pkg/config/scraper_list.go index 8fb012ed0..3ebb82586 100644 --- a/pkg/config/scraper_list.go +++ b/pkg/config/scraper_list.go @@ -3,7 +3,6 @@ package config import ( _ "embed" "encoding/json" - "io/ioutil" "os" "path/filepath" "strings" @@ -55,9 +54,9 @@ func (o *ScraperList) Load() error { fName := filepath.Join(common.AppDir, "scrapers.json") if _, err := os.Stat(fName); os.IsNotExist(err) { list, _ := json.MarshalIndent(officalScrapers, "", " ") - ioutil.WriteFile(fName, list, 0644) + os.WriteFile(fName, list, 0644) } else { - b, err := ioutil.ReadFile(fName) + b, err := os.ReadFile(fName) if err != nil { o.XbvrScrapers = officalScrapers.XbvrScrapers return err @@ -86,7 +85,7 @@ func (o *ScraperList) Load() error { list, err := json.MarshalIndent(o, "", " ") if err == nil { - ioutil.WriteFile(fName, list, 0644) + os.WriteFile(fName, list, 0644) } return nil diff --git a/pkg/dms/dlna/dms/dms.go b/pkg/dms/dlna/dms/dms.go index 454067715..4fbceafdf 100644 --- a/pkg/dms/dlna/dms/dms.go +++ b/pkg/dms/dlna/dms/dms.go @@ -7,7 +7,6 @@ import ( "errors" "fmt" "io" - "io/ioutil" "log" "net" "net/http" @@ -588,7 +587,7 @@ func (me *Server) serveIcon(w http.ResponseWriter, r *http.Request) { } defer resp.Body.Close() - bodyBytes, err := ioutil.ReadAll(resp.Body) + bodyBytes, err := io.ReadAll(resp.Body) if err != nil { return } @@ -649,13 +648,13 @@ func (server *Server) contentDirectoryInitialEvent(urls []*url.URL, sid string) continue } eventingLogger.Print(resp) - b, _ := ioutil.ReadAll(resp.Body) + b, _ := io.ReadAll(resp.Body) eventingLogger.Println(string(b)) resp.Body.Close() } } -var eventingLogger = log.New(ioutil.Discard, "", 0) +var eventingLogger = log.New(io.Discard, "", 0) func (server *Server) contentDirectoryEventSubHandler(w http.ResponseWriter, r *http.Request) { if server.StallEventSubscribe { diff --git a/pkg/dms/staticcheck.conf b/pkg/dms/staticcheck.conf new file mode 100644 index 000000000..847b77b65 --- /dev/null +++ b/pkg/dms/staticcheck.conf @@ -0,0 +1,2 @@ +# disable all checks, since this part is from another project: https://github.com/anacrolix/dms +checks = [] \ No newline at end of file diff --git a/pkg/externalreference/log.go b/pkg/externalreference/log.go index a56d98e93..affbcb90e 100644 --- a/pkg/externalreference/log.go +++ b/pkg/externalreference/log.go @@ -4,4 +4,4 @@ import ( "github.com/xbapps/xbvr/pkg/common" ) -var log = common.Log +var log = &common.Log diff --git a/pkg/externalreference/stashdb.go b/pkg/externalreference/stashdb.go index f07a803fd..22b035bb8 100644 --- a/pkg/externalreference/stashdb.go +++ b/pkg/externalreference/stashdb.go @@ -336,25 +336,6 @@ func UpdateXbvrActor(performer models.StashPerformer, xbvrActorID uint) { } } -func addToArray(existingArray string, newValue string) string { - values := []string{} - if existingArray != "" { - err := json.Unmarshal([]byte(existingArray), &values) - if err != nil { - log.Errorf("Could not extract array %s", values) - } - } - for _, existingValue := range values { - if existingValue == newValue { - return existingArray - } - } - values = append(values, newValue) - jsonBytes, _ := json.Marshal(values) - return string(jsonBytes) - -} - func convertBodyModToString(bodyMod models.StashBodyModification) string { newMod := "" diff --git a/pkg/models/db.go b/pkg/models/db.go index 33456ffee..341770afe 100644 --- a/pkg/models/db.go +++ b/pkg/models/db.go @@ -93,10 +93,8 @@ func CheckLock(lock string) bool { var obj KV err := db.Where(&KV{Key: "lock-" + lock}).First(&obj).Error - if err == nil { - return true - } - return false + + return err == nil } func RemoveLock(lock string) { diff --git a/pkg/models/model_action.go b/pkg/models/model_action.go index 61296baf2..d620a7403 100644 --- a/pkg/models/model_action.go +++ b/pkg/models/model_action.go @@ -22,8 +22,7 @@ func (a *Action) Save() { db, _ := GetDB() defer db.Close() - var err error - err = retry.Do( + var err error = retry.Do( func() error { err := db.Save(&a).Error if err != nil { diff --git a/pkg/models/model_action_actor.go b/pkg/models/model_action_actor.go index 3eb598335..e36c5510c 100644 --- a/pkg/models/model_action_actor.go +++ b/pkg/models/model_action_actor.go @@ -28,8 +28,7 @@ func (a *ActionActor) Save() { db, _ := GetDB() defer db.Close() - var err error - err = retry.Do( + var err error = retry.Do( func() error { err := db.Save(&a).Error if err != nil { diff --git a/pkg/models/model_actor.go b/pkg/models/model_actor.go index d524aeb9e..8942bff12 100644 --- a/pkg/models/model_actor.go +++ b/pkg/models/model_actor.go @@ -100,8 +100,7 @@ func (i *Actor) Save() error { db, _ := GetDB() defer db.Close() - var err error - err = retry.Do( + var err error = retry.Do( func() error { err := db.Save(&i).Error if err != nil { @@ -201,9 +200,8 @@ func QueryActors(r RequestActorList, enablePreload bool) ResponseActorList { truefalse = false fieldName = fieldName[1:] } - if strings.HasPrefix(fieldName, "&") { // & prefix indicate must have filtering - fieldName = fieldName[1:] - } + + fieldName = strings.TrimPrefix(fieldName, "&") // & prefix indicate must have filtering value := "" where := "" diff --git a/pkg/models/model_aka.go b/pkg/models/model_aka.go index 14747abbe..35b44b45f 100644 --- a/pkg/models/model_aka.go +++ b/pkg/models/model_aka.go @@ -69,11 +69,6 @@ func (o *Aka) UpdateAkaSceneCastRecords() { `) // delete scene_cast records for aka actors that have been removed - type DeleteList struct { - AkaActorId uint - SceneId uint - } - db.Exec(` with SceneIds as ( select distinct a.id, sc.scene_id diff --git a/pkg/models/model_external_reference.go b/pkg/models/model_external_reference.go index f51ff058d..2a93199ee 100644 --- a/pkg/models/model_external_reference.go +++ b/pkg/models/model_external_reference.go @@ -2,7 +2,6 @@ package models import ( "encoding/json" - "io/ioutil" "os" "path/filepath" "regexp" @@ -985,7 +984,7 @@ func (scrapeRules ActorScraperConfig) getCustomRules() { // this is to give examples, it is not loaded fName := filepath.Join(common.AppDir, "actor_scraper_config_examples.json") out, _ := json.MarshalIndent(scrapeRules, "", " ") - ioutil.WriteFile(fName, out, 0644) + os.WriteFile(fName, out, 0644) // now check if the user has any custom rules fName = filepath.Join(common.AppDir, "actor_scraper_custom_config.json") @@ -1025,12 +1024,12 @@ func (scrapeRules ActorScraperConfig) getCustomRules() { exampleConfig.StashSceneMatching["siteid"] = stashMatch out, _ := json.MarshalIndent(exampleConfig, "", " ") - ioutil.WriteFile(fName, out, 0644) + os.WriteFile(fName, out, 0644) } } else { // load any custom rules and update the builtin list var customScrapeRules ActorScraperConfig - b, err := ioutil.ReadFile(fName) + b, err := os.ReadFile(fName) if err != nil { return } diff --git a/pkg/models/model_file.go b/pkg/models/model_file.go index 6242bbda2..6fbf9df1f 100644 --- a/pkg/models/model_file.go +++ b/pkg/models/model_file.go @@ -52,8 +52,7 @@ func (f *File) Save() error { db, _ := GetDB() defer db.Close() - var err error - err = retry.Do( + var err error = retry.Do( func() error { err := db.Save(&f).Error if err != nil { diff --git a/pkg/models/model_history.go b/pkg/models/model_history.go index 482750af4..e2db0debb 100644 --- a/pkg/models/model_history.go +++ b/pkg/models/model_history.go @@ -28,8 +28,7 @@ func (o *History) Save() { db, _ := GetDB() defer db.Close() - var err error - err = retry.Do( + var err error = retry.Do( func() error { err := db.Save(&o).Error if err != nil { diff --git a/pkg/models/model_kv.go b/pkg/models/model_kv.go index 3e73118ac..53001ed31 100644 --- a/pkg/models/model_kv.go +++ b/pkg/models/model_kv.go @@ -3,7 +3,7 @@ package models import "github.com/avast/retry-go/v4" type KV struct { - Key string `json:"key" gorm:"primary_key" gorm:"unique_index"` + Key string `json:"key" gorm:"primary_key"` Value string `json:"value" sql:"type:text;"` } @@ -11,8 +11,7 @@ func (o *KV) Save() { db, _ := GetDB() defer db.Close() - var err error - err = retry.Do( + var err error = retry.Do( func() error { err := db.Save(&o).Error if err != nil { diff --git a/pkg/models/model_playlist.go b/pkg/models/model_playlist.go index 6c4957701..921156d72 100644 --- a/pkg/models/model_playlist.go +++ b/pkg/models/model_playlist.go @@ -25,8 +25,7 @@ func (o *Playlist) Save() error { db, _ := GetDB() defer db.Close() - var err error - err = retry.Do( + var err error = retry.Do( func() error { err := db.Save(&o).Error if err != nil { diff --git a/pkg/models/model_scene.go b/pkg/models/model_scene.go index aeb30fecb..62d8b225e 100644 --- a/pkg/models/model_scene.go +++ b/pkg/models/model_scene.go @@ -37,8 +37,7 @@ func (o *SceneCuepoint) Save() error { db, _ := GetDB() defer db.Close() - var err error - err = retry.Do( + var err error = retry.Do( func() error { err := db.Save(&o).Error if err != nil { @@ -137,8 +136,7 @@ func (i *Scene) Save() error { db, _ := GetDB() defer db.Close() - var err error - err = retry.Do( + var err error = retry.Do( func() error { err := db.Save(&i).Error if err != nil { @@ -351,23 +349,23 @@ func (o *Scene) UpdateStatus() { changed = true } - if scripts > 0 && o.IsScripted == false { + if scripts > 0 && !o.IsScripted { o.IsScripted = true changed = true } - if scripts == 0 && o.IsScripted == true { + if scripts == 0 && o.IsScripted { o.IsScripted = false changed = true } - if videos > 0 && o.IsAvailable == false { + if videos > 0 && !o.IsAvailable { o.IsAvailable = true o.Wishlist = false changed = true } - if videos == 0 && o.IsAvailable == true { + if videos == 0 && o.IsAvailable { o.IsAvailable = false changed = true } @@ -382,7 +380,7 @@ func (o *Scene) UpdateStatus() { changed = true } - if o.IsScripted == true { + if o.IsScripted { o.IsScripted = false changed = true } diff --git a/pkg/models/model_site.go b/pkg/models/model_site.go index cefaf0138..78b2f0b07 100644 --- a/pkg/models/model_site.go +++ b/pkg/models/model_site.go @@ -22,8 +22,7 @@ func (i *Site) Save() error { db, _ := GetDB() defer db.Close() - var err error - err = retry.Do( + var err error = retry.Do( func() error { err := db.Save(&i).Error if err != nil { @@ -53,7 +52,7 @@ func InitSites() { scrapers := GetScrapers() for i := range scrapers { - if strings.HasSuffix(scrapers[i].ID, "-single_scene") == false { + if !strings.HasSuffix(scrapers[i].ID, "-single_scene") { var st Site db.Where(&Site{ID: scrapers[i].ID}).FirstOrCreate(&st) st.Name = scrapers[i].Name diff --git a/pkg/models/model_tag.go b/pkg/models/model_tag.go index 20fe37ba1..248d400ae 100644 --- a/pkg/models/model_tag.go +++ b/pkg/models/model_tag.go @@ -19,8 +19,7 @@ func (t *Tag) Save() error { db, _ := GetDB() defer db.Close() - var err error - err = retry.Do( + var err error = retry.Do( func() error { err := db.Save(&t).Error if err != nil { diff --git a/pkg/models/model_taggroups.go b/pkg/models/model_taggroups.go index e021800d4..4723fccc3 100644 --- a/pkg/models/model_taggroups.go +++ b/pkg/models/model_taggroups.go @@ -76,11 +76,6 @@ func (o *TagGroup) UpdateSceneTagRecords() { `) // delete scene_tags for tag groups that have been removed - type DeleteList struct { - TagGroupTagId uint - SceneId uint - } - db.Exec(` with SceneIds as ( select distinct tg.id, st.scene_id diff --git a/pkg/models/model_volume.go b/pkg/models/model_volume.go index 7b54a5dea..4890adac1 100644 --- a/pkg/models/model_volume.go +++ b/pkg/models/model_volume.go @@ -62,8 +62,7 @@ func (o *Volume) Save() error { db, _ := GetDB() defer db.Close() - var err error - err = retry.Do( + var err error = retry.Do( func() error { err := db.Save(&o).Error if err != nil { diff --git a/pkg/scrape/genericactorscraper.go b/pkg/scrape/genericactorscraper.go index e872c8612..fed81a8a6 100644 --- a/pkg/scrape/genericactorscraper.go +++ b/pkg/scrape/genericactorscraper.go @@ -7,7 +7,6 @@ import ( "math" "net/http" "net/url" - "reflect" "regexp" "strconv" "strings" @@ -165,12 +164,6 @@ func GenericActorScrapersBySite(site string) { scraperConfig := models.BuildActorScraperRules() - type outputList struct { - Id uint - Url string - Linktype string - } - er := models.ExternalReference{} scrapeId := er.DetermineActorScraperBySiteId(site) @@ -295,7 +288,7 @@ func applyRules(actorPage string, source string, rules models.GenericScraperRule actor.Save() dataJson, _ := json.Marshal(data) - extrefLink := []models.ExternalReferenceLink{models.ExternalReferenceLink{InternalTable: "actors", InternalDbId: actor.ID, InternalNameId: actor.Name, ExternalSource: source, ExternalId: actorPage}} + extrefLink := []models.ExternalReferenceLink{{InternalTable: "actors", InternalDbId: actor.ID, InternalNameId: actor.Name, ExternalSource: source, ExternalId: actorPage}} extref = models.ExternalReference{ID: extref.ID, XbvrLinks: extrefLink, ExternalSource: source, ExternalId: actorPage, ExternalURL: actorPage, ExternalDate: time.Now(), ExternalData: string(dataJson)} extref.AddUpdateWithId() } else { @@ -324,23 +317,6 @@ func getSubRuleResult(rule models.GenericActorScraperRule, e *colly.HTMLElement) return result } -func checkActorUpdateRequired(linkUrl string, actor *models.Actor) bool { - db, _ := models.GetDB() - defer db.Close() - - var extRefLink models.ExternalReferenceLink - db.Preload("ExternalReference"). - Where("internal_db_id = ? and external_id = ?", actor.ID, linkUrl).First(&extRefLink) - if extRefLink.ID != 0 { - for _, scene := range actor.Scenes { - if extRefLink.ExternalReference.ExternalDate.Before(scene.CreatedAt) { - return true - } - } - } - - return true -} func assignField(field string, value string, actor *models.Actor, overwrite bool) bool { changed := false switch field { @@ -598,17 +574,3 @@ func lookupCountryCode(countryName string) (string, error) { return countries[0].Alpha2Code, nil } - -func structToMap(obj interface{}) map[string]interface{} { - values := reflect.ValueOf(obj) - typ := values.Type() - - result := make(map[string]interface{}) - for i := 0; i < values.NumField(); i++ { - key := typ.Field(i).Name - value := values.Field(i).Interface() - result[key] = value - } - - return result -} diff --git a/pkg/scrape/javlibrary.go b/pkg/scrape/javlibrary.go index bcff6342a..6638b634d 100644 --- a/pkg/scrape/javlibrary.go +++ b/pkg/scrape/javlibrary.go @@ -21,7 +21,7 @@ func ScrapeJavLibrary(out *[]models.ScrapedScene, queryString string) { if boxTitle != nil { r := regexp.MustCompile("\"([^\"]+)\" ID Search Result") match := r.FindStringSubmatch(boxTitle.Text()) - if match != nil && len(match) > 1 { + if len(match) > 1 { // Found a search results page searchQuery := strings.ToLower(match[1]) log.Printf("Search results page found for " + searchQuery) diff --git a/pkg/scrape/javutil.go b/pkg/scrape/javutil.go index 83442c915..6c701a5af 100644 --- a/pkg/scrape/javutil.go +++ b/pkg/scrape/javutil.go @@ -56,7 +56,7 @@ func ProcessJavrTag(tag string) string { // Leave out some japanese text tags matched, err := regexp.Match("[^a-z0-9_\\- /&()\\+]", []byte(taglower)) - if matched == true || err != nil { + if matched || err != nil { return "" } @@ -72,7 +72,7 @@ func determineContentId(sc *models.ScrapedScene) string { for i := range sc.Covers { href := sc.Covers[i] match := contentIdRegex.FindStringSubmatch(href) - if match != nil && len(match) > 1 { + if len(match) > 1 { contentId = match[1] log.Println("Found content ID from cover image: " + contentId) break @@ -84,7 +84,7 @@ func determineContentId(sc *models.ScrapedScene) string { for i := range sc.Gallery { href := sc.Gallery[i] match := contentIdRegex.FindStringSubmatch(href) - if match != nil && len(match) > 1 { + if len(match) > 1 { contentId = match[1] log.Println("Found content ID from gallery image: " + contentId) break @@ -105,7 +105,7 @@ func determineContentId(sc *models.ScrapedScene) string { "3dsvr": true, "fsdss": true, } - if nameMap[site] == true { + if nameMap[site] { site = "1" + site } contentId = fmt.Sprintf("%s%05d", site, i) @@ -154,9 +154,9 @@ func PostProcessJavScene(sc *models.ScrapedScene, contentId string) { // Some specific postprocessing for error-correcting 3DSVR scenes if len(contentId) > 0 && sc.Site == "DSVR" { - r := regexp.MustCompile("13dsvr0(\\d{4})") + r := regexp.MustCompile(`13dsvr0(\d{4})`) match := r.FindStringSubmatch(contentId) - if match != nil && len(match) > 1 { + if len(match) > 1 { // Found a 3DSVR scene that is being wrongly categorized as DSVR log.Println("Applying DSVR->3DSVR workaround") sid := match[1] diff --git a/pkg/scrape/lethalhardcorevr.go b/pkg/scrape/lethalhardcorevr.go index 0aa00ab5b..5c366b1a7 100644 --- a/pkg/scrape/lethalhardcorevr.go +++ b/pkg/scrape/lethalhardcorevr.go @@ -43,19 +43,6 @@ func LethalHardcoreSite(wg *sync.WaitGroup, updateSite bool, knownScenes []strin // Site ID sc.Site = siteID - if singleSceneURL != "" { - // client := resty.New() - // client.SetHeader("User-Agent", UserAgent) - // resp, err := client.R().Get("https://www.fuckpassvr.com/api/api/scene/detail") - // searchCollector.Response = resp - // if err == nil { - - // } else { - // log.Error(err) - - // } - - } // Release Date tmpDate, _ := goment.New(e.Request.Ctx.Get("date"), "MM/DD/YYYY") sc.Released = tmpDate.Format("YYYY-MM-DD") @@ -117,8 +104,6 @@ func LethalHardcoreSite(wg *sync.WaitGroup, updateSite bool, knownScenes []strin } else { if e.Attr("src") != "https://imgs1cdn.adultempire.com/res/pm/pixel.gif" { img = e.Attr("src") - } else { - log.Infof("% lethalhardcore %s style, %v ", matches) } } diff --git a/pkg/scrape/realjamvr.go b/pkg/scrape/realjamvr.go index a403a5a8b..aa162daad 100644 --- a/pkg/scrape/realjamvr.go +++ b/pkg/scrape/realjamvr.go @@ -171,9 +171,9 @@ func RealJamSite(wg *sync.WaitGroup, updateSite bool, knownScenes []string, out switch trailerId { case "": - log.Errorf("Could not determine Scene Id for %, Id not found", sc.HomepageURL) + log.Errorf("Could not determine Scene Id for %v, Id not found", sc.HomepageURL) case "mismatch": - log.Errorf("Could not determine Scene Id for %, inconsistent trailer filenames", sc.HomepageURL) + log.Errorf("Could not determine Scene Id for %v, inconsistent trailer filenames", sc.HomepageURL) default: out <- sc } @@ -186,11 +186,8 @@ func RealJamSite(wg *sync.WaitGroup, updateSite bool, knownScenes []string, out siteCollector.OnHTML(`div.panel a`, func(e *colly.HTMLElement) { sceneURL := e.Request.AbsoluteURL(e.Attr("href")) + sceneURL = strings.TrimSuffix(sceneURL, "/") - if strings.HasSuffix(sceneURL, "/") { - // make a consistent URL - sceneURL = sceneURL[0 : len(sceneURL)-1] - } // If scene exist in database, there's no need to scrape if !funk.ContainsString(knownScenes, sceneURL) && strings.Contains(sceneURL, domain+"/scene/") { sceneCollector.Visit(sceneURL) diff --git a/pkg/scrape/sexbabesvr.go b/pkg/scrape/sexbabesvr.go index d8d044c55..27d14baa7 100644 --- a/pkg/scrape/sexbabesvr.go +++ b/pkg/scrape/sexbabesvr.go @@ -14,11 +14,6 @@ import ( "github.com/xbapps/xbvr/pkg/models" ) -var ( - currentYear int - lastMonth int -) - func SexBabesVR(wg *sync.WaitGroup, updateSite bool, knownScenes []string, out chan<- models.ScrapedScene, singleSceneURL string, singeScrapeAdditionalInfo string) error { defer wg.Done() scraperID := "sexbabesvr" @@ -116,9 +111,6 @@ func SexBabesVR(wg *sync.WaitGroup, updateSite bool, knownScenes []string, out c }) }) - currentYear = time.Now().Year() - lastMonth = int(time.Now().Month()) - if singleSceneURL != "" { sceneCollector.Visit(singleSceneURL) } else { diff --git a/pkg/scrape/slrstudios.go b/pkg/scrape/slrstudios.go index 4ecd26593..d6056f92c 100644 --- a/pkg/scrape/slrstudios.go +++ b/pkg/scrape/slrstudios.go @@ -198,7 +198,7 @@ func SexLikeReal(wg *sync.WaitGroup, updateSite bool, knownScenes []string, out }) } else { // isTransScene e.ForEach("script[type=\"text/javascript\"]", func(id int, e *colly.HTMLElement) { - var re = regexp.MustCompile("videoData:\\s*(.*}),") + var re = regexp.MustCompile(`videoData:\s*(.*}),`) r := re.FindStringSubmatch(e.Text) if len(r) > 0 { JsonMetadata := strings.TrimSpace(r[1]) diff --git a/pkg/scrape/stashdb.go b/pkg/scrape/stashdb.go index 348e55c34..d9d73eb07 100644 --- a/pkg/scrape/stashdb.go +++ b/pkg/scrape/stashdb.go @@ -4,7 +4,7 @@ import ( "bytes" "encoding/json" "fmt" - "io/ioutil" + "io" "net/http" "strconv" "strings" @@ -120,7 +120,6 @@ func StashDb() { } tlog.Info("Scrape of Stashdb completed") } - return } func findStudio(studio string, field string) FindStudioResult { @@ -533,15 +532,8 @@ func callStashDb(query string, rawVariables string) []byte { defer resp.Body.Close() - bodyBytes, _ = ioutil.ReadAll(resp.Body) + bodyBytes, _ = io.ReadAll(resp.Body) return bodyBytes } return callClient() } - -func formatInternalDbId(input uint) string { - if input == 0 { - return "" - } - return strconv.FormatUint(uint64(input), 10) -} diff --git a/pkg/scrape/vr3000.go b/pkg/scrape/vr3000.go index e4e8cb032..b2bb57801 100644 --- a/pkg/scrape/vr3000.go +++ b/pkg/scrape/vr3000.go @@ -87,9 +87,8 @@ func VR3000(wg *sync.WaitGroup, updateSite bool, knownScenes []string, out chan< "%s_960_60fps_15mb_180x180_3dh.mp4", "%s_960_30fps_10mb_180x180_3dh.mp4"} for i := range filenames { - filenames[i] = fmt.Sprintf(filenames[i], sc.SiteID) + sc.Filenames = append(sc.Filenames, fmt.Sprintf(filenames[i], sc.SiteID)) } - sc.Filenames = append(filenames) } }) diff --git a/pkg/server/heatmapproxy.go b/pkg/server/heatmapproxy.go index e19d098f4..e719df2ef 100644 --- a/pkg/server/heatmapproxy.go +++ b/pkg/server/heatmapproxy.go @@ -8,7 +8,6 @@ import ( "image/jpeg" "image/png" "io" - "io/ioutil" "net/http" "net/url" "os" @@ -78,9 +77,8 @@ func getScriptFiles(urlpart string) ([]models.File, error) { return files, fmt.Errorf("scene %d has no script files", sceneId) } - for i := range scriptfiles { - files = append(files, scriptfiles[i]) - } + files = append(files, scriptfiles...) + return files, nil } @@ -214,7 +212,7 @@ func (p *HeatmapThumbnailProxy) ServeHTTP(w http.ResponseWriter, r *http.Request } p.ImageProxy.ServeHTTP(imageproxyResponseWriter, r2) - respbody, err := ioutil.ReadAll(imageproxyResponseWriter.buf) + respbody, err := io.ReadAll(imageproxyResponseWriter.buf) if err == nil { var output bytes.Buffer err = createHeatmapThumbnail(&output, bytes.NewReader(respbody), heatmapImages) diff --git a/pkg/server/server.go b/pkg/server/server.go index 327c4b681..cc66226d9 100644 --- a/pkg/server/server.go +++ b/pkg/server/server.go @@ -34,7 +34,7 @@ import ( var ( wsAddr = common.WsAddr - log = common.Log + log = &common.Log ) func authHandle(pattern string, authEnabled bool, authSecret auth.SecretProvider, handler http.Handler) { @@ -163,7 +163,7 @@ func StartServer(version, commit, branch, date string) { }, } - wampRouter, err := router.NewRouter(routerConfig, &log) + wampRouter, err := router.NewRouter(routerConfig, log) if err != nil { log.Fatal(err) } diff --git a/pkg/session/session.go b/pkg/session/session.go index 3db8b3816..17600b682 100644 --- a/pkg/session/session.go +++ b/pkg/session/session.go @@ -3,7 +3,6 @@ package session import ( "encoding/json" "fmt" - "io/ioutil" "net/url" "os" "path" @@ -78,7 +77,7 @@ func TrackSessionFromRemote(packet DeoPacket) { f := models.File{} db, _ := models.GetDB() - err = db.First(&f, currentFileID).Error + _ = db.First(&f, currentFileID).Error defer db.Close() // Create new session @@ -181,10 +180,10 @@ func dumpHeatmap(sceneID uint, data []int) error { if _, err := os.Stat(path); os.IsNotExist(err) { // Create new heatmap dataOut, _ := json.Marshal(data) - ioutil.WriteFile(path, dataOut, 0644) + os.WriteFile(path, dataOut, 0644) } else { // Update existing heatmap - b, err := ioutil.ReadFile(path) + b, err := os.ReadFile(path) if err != nil { return err } @@ -200,7 +199,7 @@ func dumpHeatmap(sceneID uint, data []int) error { } dataOut, _ := json.Marshal(data) - ioutil.WriteFile(path, dataOut, 0644) + os.WriteFile(path, dataOut, 0644) } return nil } diff --git a/pkg/tasks/content.go b/pkg/tasks/content.go index 6a612fee4..7c0dc3df1 100644 --- a/pkg/tasks/content.go +++ b/pkg/tasks/content.go @@ -4,7 +4,6 @@ import ( "encoding/json" "errors" "fmt" - "io/ioutil" "os" "path/filepath" "strconv" @@ -332,7 +331,7 @@ func Scrape(toScrape string, singleSceneURL string, singeScrapeAdditionalInfo st tlog.Infof("Scraped %v new scenes in %s", sceneCount, - time.Now().Sub(t0).Round(time.Second)) + time.Since(t0).Round(time.Second)) } } @@ -376,7 +375,7 @@ func ScrapeJAVR(queryString string, scraper string) { tlog.Infof("Scraped %v new scenes in %s", len(collectedScenes), - time.Now().Sub(t0).Round(time.Second)) + time.Since(t0).Round(time.Second)) } else { tlog.Infof("No new scenes scraped") } @@ -431,7 +430,7 @@ func ScrapeTPDB(apiToken string, sceneUrl string) { tlog.Infof("Scraped %v new scenes in %s", len(collectedScenes), - time.Now().Sub(t0).Round(time.Second)) + time.Since(t0).Round(time.Second)) } else { tlog.Infof("No new scenes scraped") } @@ -465,9 +464,9 @@ func ExportBundle() { content, err := json.MarshalIndent(out, "", " ") if err == nil { fName := filepath.Join(common.DownloadDir, fmt.Sprintf("content-bundle-v1-%v.json", time.Now().Unix())) - err = ioutil.WriteFile(fName, content, 0644) + err = os.WriteFile(fName, content, 0644) if err == nil { - tlog.Infof("Export completed in %v, file saved to %v", time.Now().Sub(t0), fName) + tlog.Infof("Export completed in %v, file saved to %v", time.Since(t0), fName) } } } @@ -517,7 +516,6 @@ func BackupBundle(inclAllSites bool, onlyIncludeOfficalSites bool, inclScenes bo var out BackupContentBundle var content []byte exportCnt := 0 - lastMessage := time.Now() if !models.CheckLock("scrape") { models.CreateLock("scrape") @@ -657,7 +655,7 @@ func BackupBundle(inclAllSites bool, onlyIncludeOfficalSites bool, inclScenes bo var externalReferences []models.ExternalReference if inclExtRefs { - lastMessage = time.Now() + lastMessage := time.Now() db.Order("external_source").Order("id").Find(&externalReferences) recCnt := 0 for idx, ref := range externalReferences { @@ -731,7 +729,7 @@ func BackupBundle(inclAllSites bool, onlyIncludeOfficalSites bool, inclScenes bo if err == nil { fName := filepath.Join(common.DownloadDir, outputBundleFilename) - err = ioutil.WriteFile(fName, content, 0644) + err = os.WriteFile(fName, content, 0644) if err == nil { tlog.Infof("Backup file generated in %v, %v scenes selected, ready to download", time.Since(t0), exportCnt) } else { diff --git a/pkg/tasks/deps.go b/pkg/tasks/deps.go index 0ca2fafc9..be6316111 100644 --- a/pkg/tasks/deps.go +++ b/pkg/tasks/deps.go @@ -99,7 +99,7 @@ func downloadFfbinaries(tool string) error { return err } - err = os.Remove(filepath.Join(common.BinDir, tool+".zip")) + _ = os.Remove(filepath.Join(common.BinDir, tool+".zip")) return nil } diff --git a/pkg/tasks/heatmap.go b/pkg/tasks/heatmap.go index eb68610ad..3f7e829a1 100644 --- a/pkg/tasks/heatmap.go +++ b/pkg/tasks/heatmap.go @@ -6,7 +6,6 @@ import ( "image" "image/draw" "image/png" - "io/ioutil" "math" "os" "path/filepath" @@ -92,7 +91,7 @@ func GenerateHeatmaps(tlog *logrus.Entry) { } func LoadFunscriptData(path string) (Script, error) { - data, err := ioutil.ReadFile(path) + data, err := os.ReadFile(path) if err != nil { return Script{}, err } diff --git a/pkg/tasks/log.go b/pkg/tasks/log.go index 284d7f105..6411806fe 100644 --- a/pkg/tasks/log.go +++ b/pkg/tasks/log.go @@ -4,4 +4,4 @@ import ( "github.com/xbapps/xbvr/pkg/common" ) -var log = common.Log +var log = &common.Log diff --git a/pkg/tasks/oshash.go b/pkg/tasks/oshash.go index f17319374..f3551e8b0 100644 --- a/pkg/tasks/oshash.go +++ b/pkg/tasks/oshash.go @@ -19,7 +19,7 @@ func HashFile(file *os.File) (hash uint64, err error) { return } if fi.Size() < ChunkSize { - return 0, fmt.Errorf("File is too small") + return 0, fmt.Errorf("file is too small") } // Read head and tail blocks. @@ -64,7 +64,7 @@ func readChunk(file *os.File, offset int64, buf []byte) (err error) { return } if n != ChunkSize { - return fmt.Errorf("Invalid read %v", n) + return fmt.Errorf("invalid read %v", n) } return } diff --git a/pkg/tray/main.go b/pkg/tray/main.go index a2b067fa7..99549279f 100644 --- a/pkg/tray/main.go +++ b/pkg/tray/main.go @@ -44,7 +44,7 @@ func onReady() { iconData, _ := ui.Assets.ReadFile("dist/icons/xbvr-128.png") systray.SetIcon(iconData) } - systray.SetTooltip(fmt.Sprintf("XBVR")) + systray.SetTooltip("XBVR") systray.AddSeparator() From 573ebb229897351b7b3f84e262fc2fff96b8b626 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Mon, 13 Nov 2023 21:02:28 +0100 Subject: [PATCH 10/33] fix(deps): update module golang.org/x/sys to v0.14.0 (#1485) Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- go.mod | 2 +- go.sum | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/go.mod b/go.mod index 88a6f8b53..c70a801a1 100644 --- a/go.mod +++ b/go.mod @@ -58,7 +58,7 @@ require ( golang.org/x/crypto v0.14.0 golang.org/x/net v0.17.0 golang.org/x/oauth2 v0.13.0 - golang.org/x/sys v0.13.0 + golang.org/x/sys v0.14.0 golang.org/x/text v0.13.0 gopkg.in/gormigrate.v1 v1.6.0 willnorris.com/go/imageproxy v0.11.3-0.20230802154432-da55ecefd3a9 diff --git a/go.sum b/go.sum index 04000aaf4..85484170a 100644 --- a/go.sum +++ b/go.sum @@ -449,6 +449,8 @@ golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.13.0 h1:Af8nKPmuFypiUBjVoU9V20FiaFXOcuZI21p0ycVYYGE= golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.14.0 h1:Vz7Qs629MkJkGyHxUlRHizWJRG2j8fbQKjELVSNhy7Q= +golang.org/x/sys v0.14.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.2.0/go.mod h1:TVmDHMZPmdnySmBfhjOoOdhjzdE1h4u1VwSiw2l1Nuc= From 3d4710debe73e9883f0a25cdae60dbcb83b0a6e6 Mon Sep 17 00:00:00 2001 From: peter23231 <148326204+peter23231@users.noreply.github.com> Date: Mon, 13 Nov 2023 21:03:38 +0100 Subject: [PATCH 11/33] prefer selected script heatmaps (#1487) --- ui/src/views/scenes/SceneCard.vue | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/ui/src/views/scenes/SceneCard.vue b/ui/src/views/scenes/SceneCard.vue index 338784283..bee487a18 100644 --- a/ui/src/views/scenes/SceneCard.vue +++ b/ui/src/views/scenes/SceneCard.vue @@ -168,8 +168,17 @@ export default { getHeatmapURL (fileId) { return `/api/dms/heatmap/${fileId}` }, - getFunscript () { - return this.item.file !== null && this.item.file.find(a => a.type === 'script' && a.has_heatmap); + getFunscript() { + if (this.item.file !== null) { + let script; + if (script = this.item.file.find((a) => a.type === 'script' && a.has_heatmap && a.is_selected_script)) { + return script + } + if (script = this.item.file.find((a) => a.type === 'script' && a.has_heatmap)) { + return script + } + } + return false; } } } From 65a5c70560e08ef5464ec7af5ddf378e91fbf0b4 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Mon, 13 Nov 2023 21:04:01 +0100 Subject: [PATCH 12/33] fix(deps): update module golang.org/x/text to v0.14.0 (#1486) Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- go.mod | 2 +- go.sum | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/go.mod b/go.mod index c70a801a1..9980e2e79 100644 --- a/go.mod +++ b/go.mod @@ -59,7 +59,7 @@ require ( golang.org/x/net v0.17.0 golang.org/x/oauth2 v0.13.0 golang.org/x/sys v0.14.0 - golang.org/x/text v0.13.0 + golang.org/x/text v0.14.0 gopkg.in/gormigrate.v1 v1.6.0 willnorris.com/go/imageproxy v0.11.3-0.20230802154432-da55ecefd3a9 ) diff --git a/go.sum b/go.sum index 85484170a..8c2c95647 100644 --- a/go.sum +++ b/go.sum @@ -469,6 +469,8 @@ golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/text v0.11.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/text v0.13.0 h1:ablQoSUd0tRdKxZewP80B+BaqeKJuVhuRxj/dkrun3k= golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= +golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= +golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/time v0.3.0 h1:rg5rLMjNzMS1RkNLzCG38eapWhnYLFYXDXj2gOlr8j4= golang.org/x/time v0.3.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= From 1371217219ec182e03f71b39ac9dafa25bc91118 Mon Sep 17 00:00:00 2001 From: toshski <104477758+toshski@users.noreply.github.com> Date: Tue, 14 Nov 2023 09:17:29 +1300 Subject: [PATCH 13/33] WetVR Scraper changes (#1488) --- pkg/migrations/migrations.go | 30 +++++++++++++++++ pkg/models/model_scene.go | 2 ++ pkg/scrape/wetvr.go | 57 +++++++++++++++++---------------- ui/src/views/scenes/Filters.vue | 1 + 4 files changed, 63 insertions(+), 27 deletions(-) diff --git a/pkg/migrations/migrations.go b/pkg/migrations/migrations.go index ef087c9aa..afa05ed4c 100644 --- a/pkg/migrations/migrations.go +++ b/pkg/migrations/migrations.go @@ -1803,6 +1803,36 @@ func Migrate() { return nil }, }, + { + ID: "0071-Update-WetVR", + Migrate: func(tx *gorm.DB) error { + var scenes []models.Scene + + err := tx.Where("site = ?", "WetVR").Find(&scenes).Error + if err != nil { + return err + } + for _, scene := range scenes { + scene.TrailerType = "scrape_html" + scene.TrailerSource = `{"scene_url":"` + scene.SceneURL + `","html_element":"deo-video source","extract_regex":"","content_base_url":"","record_path":"","content_path":"src","encoding_path":"","quality_path":"quality"}` + scene.MemberURL = strings.Replace(scene.SceneURL, "https://wetvr.com/", "https://wetvr.com/members/", 1) + + var filenames []string + err = json.Unmarshal([]byte(scene.FilenamesArr), &filenames) + baseFilename := strings.TrimPrefix(scene.SceneURL, "https://wetvr.com/video/") + if !strings.Contains(scene.FilenamesArr, "2700.mp4") { + filenames = append(filenames, "wetvr-"+baseFilename+"-2700.mp4") + filenames = append(filenames, "wetvr-"+baseFilename+"-2048.mp4") + filenames = append(filenames, "wetvr-"+baseFilename+"-1600.mp4") + filenames = append(filenames, "wetvr-"+baseFilename+"-960.mp4") + tmp, _ := json.Marshal(filenames) + scene.FilenamesArr = string(tmp) + } + tx.Save(&scene) + } + return nil + }, + }, }) if err := m.Migrate(); err != nil { diff --git a/pkg/models/model_scene.go b/pkg/models/model_scene.go index 62d8b225e..980607413 100644 --- a/pkg/models/model_scene.go +++ b/pkg/models/model_scene.go @@ -1047,6 +1047,8 @@ func queryScenes(db *gorm.DB, r RequestSceneList) (*gorm.DB, *gorm.DB) { tx = tx.Order("updated_at desc") case "script_published_desc": tx = tx.Order("script_published desc") + case "scene_id_desc": + tx = tx.Order("scene_id desc") case "random": if dbConn.Driver == "mysql" { tx = tx.Order("rand()") diff --git a/pkg/scrape/wetvr.go b/pkg/scrape/wetvr.go index 8af34c8c2..e6d864520 100644 --- a/pkg/scrape/wetvr.go +++ b/pkg/scrape/wetvr.go @@ -2,8 +2,6 @@ package scrape import ( "encoding/json" - "regexp" - "strconv" "strings" "sync" "time" @@ -24,61 +22,57 @@ func WetVR(wg *sync.WaitGroup, updateSite bool, knownScenes []string, out chan<- sceneCollector := createCollector("wetvr.com") siteCollector := createCollector("wetvr.com") - // RegEx Patterns - durationRegEx := regexp.MustCompile(`(?i)DURATION:\W(\d+)`) - - sceneCollector.OnHTML(`div#t2019`, func(e *colly.HTMLElement) { + sceneCollector.OnHTML(`div#trailer_player`, func(e *colly.HTMLElement) { sc := models.ScrapedScene{} sc.ScraperID = scraperID sc.SceneType = "VR" sc.Studio = "WetVR" sc.Site = siteID sc.HomepageURL = strings.Split(e.Request.URL.String(), "?")[0] - sc.MembersUrl = strings.Replace(sc.HomepageURL, "https://wetvr.com/", "https://members.wetvr.com/", 1) + sc.MembersUrl = strings.Replace(sc.HomepageURL, "https://wetvr.com/", "https://wetvr.com/members/", 1) // Scene ID - get from previous page sc.SiteID = e.Request.Ctx.GetAny("scene-id").(string) sc.SceneID = slugify.Slugify(sc.Site + "-" + sc.SiteID) // Title - sc.Title = strings.TrimSpace(e.ChildText(`h1.t2019-stitle`)) + sc.Title = strings.TrimSpace(e.ChildText(`div.scene-info h1`)) - // Date scenedate := e.Request.Ctx.GetAny("scene-date").(string) if scenedate != "" { - tmpDate, _ := goment.New(scenedate, "MMMM DD, YYYY") + tmpDate, _ := goment.New(scenedate, "MM/DD/YYYY") sc.Released = tmpDate.Format("YYYY-MM-DD") } - // Duration - tmpDuration := durationRegEx.FindStringSubmatch(e.ChildText(`div#t2019-stime`))[1] - sc.Duration, _ = strconv.Atoi(tmpDuration) - // Cover URLs - coverSrc := e.ChildAttr(`div#t2019-video deo-video`, "cover-image") + coverSrc := e.ChildAttr(`div[id="player-wrapper"] deo-video`, "cover-image") if coverSrc == "" { - coverSrc = e.ChildAttr(`div#t2019-video img#no-player-image`, "src") + coverSrc = strings.Split(e.ChildAttr(`div[id="no-player-wrapper"] div.bg-cover`, "style"), "background-image: url(")[1] + coverSrc = strings.TrimPrefix(coverSrc, "'") + coverSrc = strings.TrimSuffix(coverSrc, "')") } if coverSrc != "" { sc.Covers = append(sc.Covers, e.Request.AbsoluteURL(coverSrc)) } // Gallery - e.ForEach(`div.t2019-thumbs img`, func(id int, e *colly.HTMLElement) { + e.ForEach(`div.items-center a[href="/join" ] img`, func(id int, e *colly.HTMLElement) { if id > 0 { sc.Gallery = append(sc.Gallery, e.Request.AbsoluteURL(e.Attr("src"))) } }) // Synopsis - sc.Synopsis = strings.TrimSpace(e.ChildText(`div#t2019-description`)) + sc.Synopsis = strings.TrimSpace(e.ChildText(`div.items-start span`)) // trailer details - sc.TrailerType = "deovr" - sc.TrailerSrc = strings.Replace(sc.HomepageURL, "/video/", "/deovr/", 1) + sc.TrailerType = "scrape_html" + params := models.TrailerScrape{SceneUrl: sc.HomepageURL, HtmlElement: "deo-video source", ContentPath: "src", QualityPath: "quality"} + strParams, _ := json.Marshal(params) + sc.TrailerSrc = string(strParams) // Cast - e.ForEach(`div#t2019-models a`, func(id int, e *colly.HTMLElement) { + e.ForEach(`a[href^="/models/"]`, func(id int, e *colly.HTMLElement) { sc.Cast = append(sc.Cast, strings.TrimSpace(e.Text)) }) @@ -86,17 +80,21 @@ func WetVR(wg *sync.WaitGroup, updateSite bool, knownScenes []string, out chan<- // no tags on this site // Filenames - // NOTE: no way to guess filename + baseFilename := strings.TrimPrefix(sc.HomepageURL, "https://wetvr.com/video/") + sc.Filenames = append(sc.Filenames, "wetvr-"+baseFilename+"-2700.mp4") + sc.Filenames = append(sc.Filenames, "wetvr-"+baseFilename+"-2048.mp4") + sc.Filenames = append(sc.Filenames, "wetvr-"+baseFilename+"-1600.mp4") + sc.Filenames = append(sc.Filenames, "wetvr-"+baseFilename+"-960.mp4") out <- sc }) - siteCollector.OnHTML(`ul.pagination a.page-link`, func(e *colly.HTMLElement) { + siteCollector.OnHTML(`ul a.page-link`, func(e *colly.HTMLElement) { pageURL := e.Request.AbsoluteURL(e.Attr("href")) siteCollector.Visit(pageURL) }) - siteCollector.OnHTML(`div.card`, func(e *colly.HTMLElement) { + siteCollector.OnHTML(`div:has(p:contains("Latest")) div[id^="r-"]`, func(e *colly.HTMLElement) { sceneURL := e.Request.AbsoluteURL(e.ChildAttr("a", "href")) // If scene exist in database, there's no need to scrape @@ -104,9 +102,14 @@ func WetVR(wg *sync.WaitGroup, updateSite bool, knownScenes []string, out chan<- // SceneID and release date are only available here on div.card ctx := colly.NewContext() - ctx.Put("scene-id", e.Attr("data-video-id")) - ctx.Put("scene-date", e.Attr("data-date")) - + ctx.Put("scene-id", strings.TrimPrefix(e.Attr("id"), "r-")) + // get the date if it exists + pDate := e.DOM.Find(`div.video-thumbnail-footer div>span`) + if pDate.Length() > 0 { + ctx.Put("scene-date", pDate.Text()) + } else { + ctx.Put("scene-date", "") + } sceneCollector.Request("GET", sceneURL, nil, ctx, nil) } }) diff --git a/ui/src/views/scenes/Filters.vue b/ui/src/views/scenes/Filters.vue index 04b3e6538..a6738904b 100644 --- a/ui/src/views/scenes/Filters.vue +++ b/ui/src/views/scenes/Filters.vue @@ -56,6 +56,7 @@ + From ea7b235fdb8df6c98cedcf07ac0a91db67615339 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Mon, 13 Nov 2023 21:17:49 +0100 Subject: [PATCH 14/33] chore(deps): update babel monorepo to v7.23.3 (#1490) Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- package.json | 4 +-- yarn.lock | 74 ++++++++++++++++++++++++++++++++++++++++------------ 2 files changed, 59 insertions(+), 19 deletions(-) diff --git a/package.json b/package.json index b7300701d..8185a6526 100644 --- a/package.json +++ b/package.json @@ -35,8 +35,8 @@ "wampy": "6.4.2" }, "devDependencies": { - "@babel/core": "7.23.2", - "@babel/eslint-parser": "7.22.15", + "@babel/core": "7.23.3", + "@babel/eslint-parser": "7.23.3", "@vue/cli-plugin-babel": "5.0.8", "@vue/cli-plugin-eslint": "5.0.8", "@vue/cli-service": "5.0.8", diff --git a/yarn.lock b/yarn.lock index b27f97adf..e943b4cd7 100644 --- a/yarn.lock +++ b/yarn.lock @@ -69,21 +69,21 @@ resolved "https://registry.yarnpkg.com/@babel/compat-data/-/compat-data-7.22.9.tgz#71cdb00a1ce3a329ce4cbec3a44f9fef35669730" integrity sha512-5UamI7xkUcJ3i9qVDS+KFDEK8/7oJ55/sJMB1Ge7IEapr7KfdfV/HErR+koZwOfd+SgtFKOKRhRakdg++DcJpQ== -"@babel/core@7.23.2": - version "7.23.2" - resolved "https://registry.yarnpkg.com/@babel/core/-/core-7.23.2.tgz#ed10df0d580fff67c5f3ee70fd22e2e4c90a9f94" - integrity sha512-n7s51eWdaWZ3vGT2tD4T7J6eJs3QoBXydv7vkUM06Bf1cbVD2Kc2UrkzhiQwobfV7NwOnQXYL7UBJ5VPU+RGoQ== +"@babel/core@7.23.3": + version "7.23.3" + resolved "https://registry.yarnpkg.com/@babel/core/-/core-7.23.3.tgz#5ec09c8803b91f51cc887dedc2654a35852849c9" + integrity sha512-Jg+msLuNuCJDyBvFv5+OKOUjWMZgd85bKjbICd3zWrKAo+bJ49HJufi7CQE0q0uR8NGyO6xkCACScNqyjHSZew== dependencies: "@ampproject/remapping" "^2.2.0" "@babel/code-frame" "^7.22.13" - "@babel/generator" "^7.23.0" + "@babel/generator" "^7.23.3" "@babel/helper-compilation-targets" "^7.22.15" - "@babel/helper-module-transforms" "^7.23.0" + "@babel/helper-module-transforms" "^7.23.3" "@babel/helpers" "^7.23.2" - "@babel/parser" "^7.23.0" + "@babel/parser" "^7.23.3" "@babel/template" "^7.22.15" - "@babel/traverse" "^7.23.2" - "@babel/types" "^7.23.0" + "@babel/traverse" "^7.23.3" + "@babel/types" "^7.23.3" convert-source-map "^2.0.0" debug "^4.1.0" gensync "^1.0.0-beta.2" @@ -111,10 +111,10 @@ json5 "^2.2.1" semver "^6.3.0" -"@babel/eslint-parser@7.22.15": - version "7.22.15" - resolved "https://registry.yarnpkg.com/@babel/eslint-parser/-/eslint-parser-7.22.15.tgz#263f059c476e29ca4972481a17b8b660cb025a34" - integrity sha512-yc8OOBIQk1EcRrpizuARSQS0TWAcOMpEJ1aafhNznaeYkeL+OhqnDObGFylB8ka8VFF/sZc+S4RzHyO+3LjQxg== +"@babel/eslint-parser@7.23.3": + version "7.23.3" + resolved "https://registry.yarnpkg.com/@babel/eslint-parser/-/eslint-parser-7.23.3.tgz#7bf0db1c53b54da0c8a12627373554a0828479ca" + integrity sha512-9bTuNlyx7oSstodm1cR1bECj4fkiknsDa1YniISkJemMY3DGhJNYBECbe6QD/q54mp2J8VO66jW3/7uP//iFCw== dependencies: "@nicolo-ribaudo/eslint-scope-5-internals" "5.1.1-v1" eslint-visitor-keys "^2.1.0" @@ -139,6 +139,16 @@ "@jridgewell/trace-mapping" "^0.3.17" jsesc "^2.5.1" +"@babel/generator@^7.23.3": + version "7.23.3" + resolved "https://registry.yarnpkg.com/@babel/generator/-/generator-7.23.3.tgz#86e6e83d95903fbe7613f448613b8b319f330a8e" + integrity sha512-keeZWAV4LU3tW0qRi19HRpabC/ilM0HRBBzf9/k8FFiG4KVpiv0FIy4hHfLfFQZNhziCTPTmd59zoyv6DNISzg== + dependencies: + "@babel/types" "^7.23.3" + "@jridgewell/gen-mapping" "^0.3.2" + "@jridgewell/trace-mapping" "^0.3.17" + jsesc "^2.5.1" + "@babel/helper-annotate-as-pure@^7.12.13": version "7.12.13" resolved "https://registry.yarnpkg.com/@babel/helper-annotate-as-pure/-/helper-annotate-as-pure-7.12.13.tgz#0f58e86dfc4bb3b1fcd7db806570e177d439b6ab" @@ -324,10 +334,10 @@ "@babel/traverse" "^7.18.6" "@babel/types" "^7.18.6" -"@babel/helper-module-transforms@^7.23.0": - version "7.23.0" - resolved "https://registry.yarnpkg.com/@babel/helper-module-transforms/-/helper-module-transforms-7.23.0.tgz#3ec246457f6c842c0aee62a01f60739906f7047e" - integrity sha512-WhDWw1tdrlT0gMgUJSlX0IQvoO1eN279zrAUbVB+KpV2c3Tylz8+GnKOLllCS6Z/iZQEyVYxhZVUdPTqs2YYPw== +"@babel/helper-module-transforms@^7.23.3": + version "7.23.3" + resolved "https://registry.yarnpkg.com/@babel/helper-module-transforms/-/helper-module-transforms-7.23.3.tgz#d7d12c3c5d30af5b3c0fcab2a6d5217773e2d0f1" + integrity sha512-7bBs4ED9OmswdfDzpz4MpWgSrV7FXlc3zIagvLFjS5H+Mk7Snr21vQ6QwrsoCGMfNC4e4LQPdoULEt4ykz0SRQ== dependencies: "@babel/helper-environment-visitor" "^7.22.20" "@babel/helper-module-imports" "^7.22.15" @@ -518,6 +528,11 @@ resolved "https://registry.yarnpkg.com/@babel/parser/-/parser-7.23.0.tgz#da950e622420bf96ca0d0f2909cdddac3acd8719" integrity sha512-vvPKKdMemU85V9WE/l5wZEmImpCtLqbnTvqDS2U1fJ96KrxoW7KrXhNsNCblQlg8Ck4b85yxdTyelsMUgFUXiw== +"@babel/parser@^7.23.3": + version "7.23.3" + resolved "https://registry.yarnpkg.com/@babel/parser/-/parser-7.23.3.tgz#0ce0be31a4ca4f1884b5786057cadcb6c3be58f9" + integrity sha512-uVsWNvlVsIninV2prNz/3lHCb+5CJ+e+IUBfbjToAHODtfGYLfCFuY4AU7TskI+dAKk+njsPiBjq1gKTvZOBaw== + "@babel/plugin-bugfix-safari-id-destructuring-collision-in-function-expression@^7.18.6": version "7.18.6" resolved "https://registry.yarnpkg.com/@babel/plugin-bugfix-safari-id-destructuring-collision-in-function-expression/-/plugin-bugfix-safari-id-destructuring-collision-in-function-expression-7.18.6.tgz#da5b8f9a580acdfbe53494dba45ea389fb09a4d2" @@ -1236,6 +1251,22 @@ debug "^4.1.0" globals "^11.1.0" +"@babel/traverse@^7.23.3": + version "7.23.3" + resolved "https://registry.yarnpkg.com/@babel/traverse/-/traverse-7.23.3.tgz#26ee5f252e725aa7aca3474aa5b324eaf7908b5b" + integrity sha512-+K0yF1/9yR0oHdE0StHuEj3uTPzwwbrLGfNOndVJVV2TqA5+j3oljJUb4nmB954FLGjNem976+B+eDuLIjesiQ== + dependencies: + "@babel/code-frame" "^7.22.13" + "@babel/generator" "^7.23.3" + "@babel/helper-environment-visitor" "^7.22.20" + "@babel/helper-function-name" "^7.23.0" + "@babel/helper-hoist-variables" "^7.22.5" + "@babel/helper-split-export-declaration" "^7.22.6" + "@babel/parser" "^7.23.3" + "@babel/types" "^7.23.3" + debug "^4.1.0" + globals "^11.1.0" + "@babel/types@^7.0.0", "@babel/types@^7.12.13", "@babel/types@^7.13.12", "@babel/types@^7.4.4": version "7.14.0" resolved "https://registry.yarnpkg.com/@babel/types/-/types-7.14.0.tgz#3fc3fc74e0cdad878182e5f66cc6bcab1915a802" @@ -1270,6 +1301,15 @@ "@babel/helper-validator-identifier" "^7.22.5" to-fast-properties "^2.0.0" +"@babel/types@^7.23.3": + version "7.23.3" + resolved "https://registry.yarnpkg.com/@babel/types/-/types-7.23.3.tgz#d5ea892c07f2ec371ac704420f4dcdb07b5f9598" + integrity sha512-OZnvoH2l8PK5eUvEcUyCt/sXgr/h+UWpVuBbOljwcrAgUl6lpchoQ++PHGyQy1AtYnVA6CEq3y5xeEI10brpXw== + dependencies: + "@babel/helper-string-parser" "^7.22.5" + "@babel/helper-validator-identifier" "^7.22.20" + to-fast-properties "^2.0.0" + "@eslint-community/eslint-utils@^4.2.0": version "4.2.0" resolved "https://registry.yarnpkg.com/@eslint-community/eslint-utils/-/eslint-utils-4.2.0.tgz#a831e6e468b4b2b5ae42bf658bea015bf10bc518" From 5f23a0647d95c8dda1ca401d6bde66f433f4c503 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Mon, 13 Nov 2023 21:18:10 +0100 Subject: [PATCH 15/33] fix(deps): update module github.com/xo/dburl to v0.17.1 (#1491) Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- go.mod | 2 +- go.sum | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/go.mod b/go.mod index 9980e2e79..405981f0f 100644 --- a/go.mod +++ b/go.mod @@ -54,7 +54,7 @@ require ( github.com/thoas/go-funk v0.9.3 github.com/tidwall/gjson v1.17.0 github.com/x-cray/logrus-prefixed-formatter v0.5.2 - github.com/xo/dburl v0.17.0 + github.com/xo/dburl v0.17.1 golang.org/x/crypto v0.14.0 golang.org/x/net v0.17.0 golang.org/x/oauth2 v0.13.0 diff --git a/go.sum b/go.sum index 8c2c95647..ebf7edcb3 100644 --- a/go.sum +++ b/go.sum @@ -370,6 +370,8 @@ github.com/xo/dburl v0.16.0 h1:jlBeGe8fnsW+vBYemte903WHQbJnZx7OpJZy2ofq+5g= github.com/xo/dburl v0.16.0/go.mod h1:B7/G9FGungw6ighV8xJNwWYQPMfn3gsi2sn5SE8Bzco= github.com/xo/dburl v0.17.0 h1:sUgx24FNMbdxjutH6r4eg4y5S75RMjb/C+eGqnttcqU= github.com/xo/dburl v0.17.0/go.mod h1:B7/G9FGungw6ighV8xJNwWYQPMfn3gsi2sn5SE8Bzco= +github.com/xo/dburl v0.17.1 h1:n8jpWqYwe8TJQ1n8OvJuKDP9GxnPS+52T/xQtjADTIM= +github.com/xo/dburl v0.17.1/go.mod h1:B7/G9FGungw6ighV8xJNwWYQPMfn3gsi2sn5SE8Bzco= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= go.etcd.io/bbolt v1.3.7 h1:j+zJOnnEjF/kyHlDDgGnVL/AIqIJPq8UoB2GSNfkUfQ= go.etcd.io/bbolt v1.3.7/go.mod h1:N9Mkw9X8x5fupy0IKsmuqVtoGDyxsaDlbk4Rd05IAQw= From 07997438bb489dcb36c89d7205e6992a71f57e64 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Tue, 14 Nov 2023 21:25:31 +0100 Subject: [PATCH 16/33] fix(deps): update module golang.org/x/crypto to v0.15.0 (#1492) Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- go.mod | 4 ++-- go.sum | 4 ++++ 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/go.mod b/go.mod index 405981f0f..c86ad978e 100644 --- a/go.mod +++ b/go.mod @@ -55,7 +55,7 @@ require ( github.com/tidwall/gjson v1.17.0 github.com/x-cray/logrus-prefixed-formatter v0.5.2 github.com/xo/dburl v0.17.1 - golang.org/x/crypto v0.14.0 + golang.org/x/crypto v0.15.0 golang.org/x/net v0.17.0 golang.org/x/oauth2 v0.13.0 golang.org/x/sys v0.14.0 @@ -143,7 +143,7 @@ require ( github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8 // indirect go.etcd.io/bbolt v1.3.7 // indirect golang.org/x/image v0.10.0 // indirect - golang.org/x/term v0.13.0 // indirect + golang.org/x/term v0.14.0 // indirect google.golang.org/appengine v1.6.7 // indirect google.golang.org/protobuf v1.31.0 // indirect gopkg.in/sourcemap.v1 v1.0.5 // indirect diff --git a/go.sum b/go.sum index ebf7edcb3..d5d013f4a 100644 --- a/go.sum +++ b/go.sum @@ -385,6 +385,8 @@ golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPh golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.14.0 h1:wBqGXzWJW6m1XrIKlAH0Hs1JJ7+9KBwnIO8v66Q9cHc= golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4= +golang.org/x/crypto v0.15.0 h1:frVn1TEaCEaZcn3Tmd7Y2b5KKPaZ+I32Q2OA3kYp5TA= +golang.org/x/crypto v0.15.0/go.mod h1:4ChreQoLWfG3xLDer1WdlH5NdlQ3+mwnQq1YTKY+72g= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/image v0.0.0-20191009234506-e7c1f5e7dbb8/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= golang.org/x/image v0.10.0 h1:gXjUUtwtx5yOE0VKWq1CH4IJAClq4UGgUA3i+rpON9M= @@ -460,6 +462,8 @@ golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= golang.org/x/term v0.13.0 h1:bb+I9cTfFazGW51MZqBVmZy7+JEJMouUHTUSKVQLBek= golang.org/x/term v0.13.0/go.mod h1:LTmsnFJwVN6bCy1rVCoS+qHT1HhALEFxKncY3WNNh4U= +golang.org/x/term v0.14.0 h1:LGK9IlZ8T9jvdy6cTdfKUCltatMFOehAQo9SRC46UQ8= +golang.org/x/term v0.14.0/go.mod h1:TySc+nGkYR6qt8km8wUhuFRTVSMIX3XPR58y2lC8vww= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= From e96289351373364cd55779af452a8e80fa1d40bf Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Tue, 14 Nov 2023 21:27:23 +0100 Subject: [PATCH 17/33] fix(deps): update module golang.org/x/net to v0.18.0 (#1493) Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- go.mod | 2 +- go.sum | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/go.mod b/go.mod index c86ad978e..8b2e0fbbb 100644 --- a/go.mod +++ b/go.mod @@ -56,7 +56,7 @@ require ( github.com/x-cray/logrus-prefixed-formatter v0.5.2 github.com/xo/dburl v0.17.1 golang.org/x/crypto v0.15.0 - golang.org/x/net v0.17.0 + golang.org/x/net v0.18.0 golang.org/x/oauth2 v0.13.0 golang.org/x/sys v0.14.0 golang.org/x/text v0.14.0 diff --git a/go.sum b/go.sum index d5d013f4a..d80e633c3 100644 --- a/go.sum +++ b/go.sum @@ -420,6 +420,8 @@ golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= golang.org/x/net v0.17.0 h1:pVaXccu2ozPjCXewfr1S7xza/zcXTity9cCdXQYSjIM= golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE= +golang.org/x/net v0.18.0 h1:mIYleuAkSbHh0tCv7RvjL3F6ZVbLjq4+R7zbOn3Kokg= +golang.org/x/net v0.18.0/go.mod h1:/czyP5RqHAH4odGYxBJ1qz0+CE5WZ+2j1YgoEo8F2jQ= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.2.0/go.mod h1:Cwn6afJ8jrQwYMxQDTpISoXmXW9I6qF6vDeuuoX3Ibs= golang.org/x/oauth2 v0.11.0 h1:vPL4xzxBM4niKCW6g9whtaWVXTJf1U5e4aZxxFx/gbU= From 7e0314bf857856139ecf3a10efa0015960388b07 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Tue, 14 Nov 2023 21:27:48 +0100 Subject: [PATCH 18/33] fix(deps): update willnorris.com/go/imageproxy digest to ef50c1f (#1495) Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- go.mod | 2 +- go.sum | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/go.mod b/go.mod index 8b2e0fbbb..6dd32adf7 100644 --- a/go.mod +++ b/go.mod @@ -61,7 +61,7 @@ require ( golang.org/x/sys v0.14.0 golang.org/x/text v0.14.0 gopkg.in/gormigrate.v1 v1.6.0 - willnorris.com/go/imageproxy v0.11.3-0.20230802154432-da55ecefd3a9 + willnorris.com/go/imageproxy v0.11.3-0.20231113231555-ef50c1f9a64e ) require ( diff --git a/go.sum b/go.sum index d80e633c3..03e1d0351 100644 --- a/go.sum +++ b/go.sum @@ -545,3 +545,5 @@ willnorris.com/go/gifresize v1.0.0 h1:GKS68zjNhHMqkgNTv4iFAO/j/sNcVSOHQ7SqmDAIAm willnorris.com/go/gifresize v1.0.0/go.mod h1:eBM8gogBGCcaH603vxSpnfjwXIpq6nmnj/jauBDKtAk= willnorris.com/go/imageproxy v0.11.3-0.20230802154432-da55ecefd3a9 h1:q8x9zY+f2kiW9B2C7bseFNW44w4kKeAu9eoCplqeeZw= willnorris.com/go/imageproxy v0.11.3-0.20230802154432-da55ecefd3a9/go.mod h1:hDdpt+NIOi8OsV4fa9pFjIpo/7DXd0in7wOF22Az/o0= +willnorris.com/go/imageproxy v0.11.3-0.20231113231555-ef50c1f9a64e h1:TF6FfnbyfLJJG3/xHBxbZralOo+wB0LMD0fFz30G9YU= +willnorris.com/go/imageproxy v0.11.3-0.20231113231555-ef50c1f9a64e/go.mod h1:dWeBm4q0BA+XSs5ZEp0Aab4M55uu1daFMrbewDqEO1w= From e2cbdcd7cf51244f9e130a4e84ab468ac012e1b1 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Tue, 14 Nov 2023 21:29:53 +0100 Subject: [PATCH 19/33] chore(deps): update actions/checkout digest to b4ffde6 (#1427) Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- .github/workflows/lint.yml | 2 +- .github/workflows/release.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index a45850704..f20d267a4 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -23,7 +23,7 @@ jobs: node-version: '12' - name: Checkout code - uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac # v4 + uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4 - name: Check code formatting using gofmt uses: Jerome1337/gofmt-action@v1.0.5 diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index d7ba5648f..238a8ae87 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -16,7 +16,7 @@ jobs: steps: - name: Checkout code - uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac # v4 + uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4 with: submodules: 'true' fetch-depth: 0 From 77e3f19720d67d0f09fdbcdb41f238cd1d83f967 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Tue, 14 Nov 2023 21:30:20 +0100 Subject: [PATCH 20/33] fix(deps): update module github.com/avast/retry-go/v4 to v4.5.1 (#1496) Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- go.mod | 2 +- go.sum | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/go.mod b/go.mod index 6dd32adf7..2af67a9f9 100644 --- a/go.mod +++ b/go.mod @@ -8,7 +8,7 @@ require ( github.com/abbot/go-http-auth v0.4.0 github.com/anacrolix/ffprobe v1.1.0 github.com/araddon/dateparse v0.0.0-20210429162001-6b43995a97de - github.com/avast/retry-go/v4 v4.5.0 + github.com/avast/retry-go/v4 v4.5.1 github.com/blevesearch/bleve/v2 v2.3.10 github.com/bregydoc/gtranslate v0.0.0-20200913051839-1bd07f6c1fc5 github.com/creasty/defaults v1.7.0 diff --git a/go.sum b/go.sum index 03e1d0351..f33881a88 100644 --- a/go.sum +++ b/go.sum @@ -34,6 +34,8 @@ github.com/araddon/dateparse v0.0.0-20210429162001-6b43995a97de h1:FxWPpzIjnTlhP github.com/araddon/dateparse v0.0.0-20210429162001-6b43995a97de/go.mod h1:DCaWoUhZrYW9p1lxo/cm8EmUOOzAPSEZNGF2DK1dJgw= github.com/avast/retry-go/v4 v4.5.0 h1:QoRAZZ90cj5oni2Lsgl2GW8mNTnUCnmpx/iKpwVisHg= github.com/avast/retry-go/v4 v4.5.0/go.mod h1:7hLEXp0oku2Nir2xBAsg0PTphp9z71bN5Aq1fboC3+I= +github.com/avast/retry-go/v4 v4.5.1 h1:AxIx0HGi4VZ3I02jr78j5lZ3M6x1E0Ivxa6b0pUUh7o= +github.com/avast/retry-go/v4 v4.5.1/go.mod h1:/sipNsvNB3RRuT5iNcb6h73nw3IBmXJ/H3XrCQYSOpc= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/bits-and-blooms/bitset v1.2.0 h1:Kn4yilvwNtMACtf1eYDlG8H77R07mZSPbMjLyS07ChA= From 6e901713643f1b8bdeb1a226a9623f5c9d2351ed Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Tue, 14 Nov 2023 22:52:34 +0100 Subject: [PATCH 21/33] fix(deps): update module golang.org/x/oauth2 to v0.14.0 (#1499) Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- go.mod | 2 +- go.sum | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/go.mod b/go.mod index 2af67a9f9..f2346e648 100644 --- a/go.mod +++ b/go.mod @@ -57,7 +57,7 @@ require ( github.com/xo/dburl v0.17.1 golang.org/x/crypto v0.15.0 golang.org/x/net v0.18.0 - golang.org/x/oauth2 v0.13.0 + golang.org/x/oauth2 v0.14.0 golang.org/x/sys v0.14.0 golang.org/x/text v0.14.0 gopkg.in/gormigrate.v1 v1.6.0 diff --git a/go.sum b/go.sum index f33881a88..220af14e6 100644 --- a/go.sum +++ b/go.sum @@ -430,6 +430,8 @@ golang.org/x/oauth2 v0.11.0 h1:vPL4xzxBM4niKCW6g9whtaWVXTJf1U5e4aZxxFx/gbU= golang.org/x/oauth2 v0.11.0/go.mod h1:LdF7O/8bLR/qWK9DrpXmbHLTouvRHK0SgJl0GmDBchk= golang.org/x/oauth2 v0.13.0 h1:jDDenyj+WgFtmV3zYVoi8aE2BwtXFLWOA67ZfNWftiY= golang.org/x/oauth2 v0.13.0/go.mod h1:/JMhi4ZRXAf4HG9LiNmxvk+45+96RUlVThiH8FzNBn0= +golang.org/x/oauth2 v0.14.0 h1:P0Vrf/2538nmC0H+pEQ3MNFRRnVR7RlqyVw+bvm26z0= +golang.org/x/oauth2 v0.14.0/go.mod h1:lAtNWgaWfL4cm7j2OV8TxGi9Qb7ECORx8DktCY74OwM= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= From fa96d7731c79cc9a30be0cd54be26e6eda2ea221 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Fri, 24 Nov 2023 11:48:42 +0100 Subject: [PATCH 22/33] fix(deps): update module github.com/xo/dburl to v0.18.2 (#1498) Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- go.mod | 2 +- go.sum | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/go.mod b/go.mod index f2346e648..959c9b6b1 100644 --- a/go.mod +++ b/go.mod @@ -54,7 +54,7 @@ require ( github.com/thoas/go-funk v0.9.3 github.com/tidwall/gjson v1.17.0 github.com/x-cray/logrus-prefixed-formatter v0.5.2 - github.com/xo/dburl v0.17.1 + github.com/xo/dburl v0.18.2 golang.org/x/crypto v0.15.0 golang.org/x/net v0.18.0 golang.org/x/oauth2 v0.14.0 diff --git a/go.sum b/go.sum index 220af14e6..5c788e01b 100644 --- a/go.sum +++ b/go.sum @@ -374,6 +374,8 @@ github.com/xo/dburl v0.17.0 h1:sUgx24FNMbdxjutH6r4eg4y5S75RMjb/C+eGqnttcqU= github.com/xo/dburl v0.17.0/go.mod h1:B7/G9FGungw6ighV8xJNwWYQPMfn3gsi2sn5SE8Bzco= github.com/xo/dburl v0.17.1 h1:n8jpWqYwe8TJQ1n8OvJuKDP9GxnPS+52T/xQtjADTIM= github.com/xo/dburl v0.17.1/go.mod h1:B7/G9FGungw6ighV8xJNwWYQPMfn3gsi2sn5SE8Bzco= +github.com/xo/dburl v0.18.2 h1:9xqcVf+JEV7bcUa1OjCsoax06roohYFdye6xkvBKo50= +github.com/xo/dburl v0.18.2/go.mod h1:B7/G9FGungw6ighV8xJNwWYQPMfn3gsi2sn5SE8Bzco= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= go.etcd.io/bbolt v1.3.7 h1:j+zJOnnEjF/kyHlDDgGnVL/AIqIJPq8UoB2GSNfkUfQ= go.etcd.io/bbolt v1.3.7/go.mod h1:N9Mkw9X8x5fupy0IKsmuqVtoGDyxsaDlbk4Rd05IAQw= From 1dc721d8a1ae12950670f74c13b751b60524dd4e Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Fri, 24 Nov 2023 11:49:30 +0100 Subject: [PATCH 23/33] chore(deps): update docker/setup-qemu-action action to v3 (#1502) Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- .github/workflows/release.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 238a8ae87..336766898 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -22,7 +22,7 @@ jobs: fetch-depth: 0 - name: Set up QEMU - uses: docker/setup-qemu-action@v2 + uses: docker/setup-qemu-action@v3 - name: Build UI run: cd ui && yarn install && yarn build From 624b02510c6e05807999a09ec0e857e7ad22b165 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Fri, 24 Nov 2023 13:07:34 +0100 Subject: [PATCH 24/33] chore(deps): update actions/setup-node action to v4 (#1501) Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- .github/workflows/lint.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index f20d267a4..982a58c41 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -18,7 +18,7 @@ jobs: go-version: 1.20.x - name: Install Node.js - uses: actions/setup-node@v3 + uses: actions/setup-node@v4 with: node-version: '12' From 1ca8d104cff76622a674c7e2b5f2cc21dbde12e4 Mon Sep 17 00:00:00 2001 From: toshski <104477758+toshski@users.noreply.github.com> Date: Sat, 25 Nov 2023 01:08:16 +1300 Subject: [PATCH 25/33] fix: exclude hidden scenes from Actor average rating (#1503) --- pkg/models/model_actor.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/pkg/models/model_actor.go b/pkg/models/model_actor.go index 8942bff12..f7c37d1b3 100644 --- a/pkg/models/model_actor.go +++ b/pkg/models/model_actor.go @@ -381,10 +381,10 @@ func QueryActors(r RequestActorList, enablePreload bool) ResponseActorList { tx = tx.Where("actors.star_rating <= ?", r.MaxRating.OrElse(5)) } if r.MinSceneRating.OrElse(0) > 0 { - tx = tx.Where("(select AVG(s.star_rating) from scene_cast sc join scenes s on s.id=sc.scene_id where sc.actor_id =actors.id and s.star_rating > 0 ) >= ?", r.MinSceneRating.OrElse(0)) + tx = tx.Where("(select AVG(s.star_rating) from scene_cast sc join scenes s on s.id=sc.scene_id where sc.actor_id =actors.id and s.star_rating > 0 and is_hidden=0) >= ?", r.MinSceneRating.OrElse(0)) } if r.MaxSceneRating.OrElse(5) < 5 { - tx = tx.Where("(select AVG(s.star_rating) from scene_cast sc join scenes s on s.id=sc.scene_id where sc.actor_id =actors.id and s.star_rating > 0 ) <= ?", r.MaxSceneRating.OrElse(50)) + tx = tx.Where("(select AVG(s.star_rating) from scene_cast sc join scenes s on s.id=sc.scene_id where sc.actor_id =actors.id and s.star_rating > 0 and is_hidden=0) <= ?", r.MaxSceneRating.OrElse(50)) } var sites []string var mustHaveSites []string @@ -432,7 +432,7 @@ func QueryActors(r RequestActorList, enablePreload bool) ResponseActorList { Order("actors.star_rating asc") case "scene_rating_desc": tx = tx. - Order("(select AVG(s.star_rating) from scene_cast sc join scenes s on s.id=sc.scene_id where sc.actor_id =actors.id and s.star_rating > 0 ) desc, (select count(*) from scene_cast sc join scenes s on s.id=sc.scene_id where sc.actor_id =actors.id and s.star_rating > 0 ) desc, (select count(*) from scene_cast sc join scenes s on s.id=sc.scene_id where sc.actor_id =actors.id) desc") + Order("(select AVG(s.star_rating) from scene_cast sc join scenes s on s.id=sc.scene_id where sc.actor_id =actors.id and s.star_rating > 0 and is_hidden=0) desc, (select count(*) from scene_cast sc join scenes s on s.id=sc.scene_id where sc.actor_id =actors.id and s.star_rating > 0 and is_hidden=0) desc, (select count(*) from scene_cast sc join scenes s on s.id=sc.scene_id where sc.actor_id =actors.id) desc") case "scene_release_desc": tx = tx. Order("IFNULL((select max(s.release_date) from scene_cast sc join scenes s on s.id=sc.scene_id where sc.actor_id =actors.id),'1970-01-01') DESC, actors.avail_count desc, actors.`count` desc") @@ -493,7 +493,7 @@ func QueryActors(r RequestActorList, enablePreload bool) ResponseActorList { out.Offset = offset tx = tx.Select(`distinct actors.*, - (select AVG(s.star_rating) scene_avg from scene_cast sc join scenes s on s.id=sc.scene_id where sc.actor_id =actors.id and s.star_rating > 0 ) as scene_rating_average + (select AVG(s.star_rating) scene_avg from scene_cast sc join scenes s on s.id=sc.scene_id where sc.actor_id =actors.id and s.star_rating > 0 and is_hidden=0) as scene_rating_average `) tx.Limit(limit). @@ -531,7 +531,7 @@ func (o *Actor) GetIfExistByPKWithSceneAvg(id uint) error { tx := db.Model(&Actor{}) tx = tx.Select(`actors.*, - (select AVG(s.star_rating) scene_avg from scene_cast sc join scenes s on s.id=sc.scene_id where sc.actor_id =actors.id and s.star_rating > 0 ) as scene_rating_average`) + (select AVG(s.star_rating) scene_avg from scene_cast sc join scenes s on s.id=sc.scene_id where sc.actor_id =actors.id and s.star_rating > 0 and is_hidden=0) as scene_rating_average`) return tx. Preload("Scenes", func(db *gorm.DB) *gorm.DB { From e6f38b6c67f7dc6293b76bfd087ded9810506c09 Mon Sep 17 00:00:00 2001 From: toshski <104477758+toshski@users.noreply.github.com> Date: Sat, 25 Nov 2023 01:09:19 +1300 Subject: [PATCH 26/33] scrapers: Fix duration for VRPorn scrapers (#1509) --- pkg/scrape/vrporn.go | 15 ++++----------- 1 file changed, 4 insertions(+), 11 deletions(-) diff --git a/pkg/scrape/vrporn.go b/pkg/scrape/vrporn.go index e05efc9c7..b8b7a78f3 100644 --- a/pkg/scrape/vrporn.go +++ b/pkg/scrape/vrporn.go @@ -24,7 +24,6 @@ func VRPorn(wg *sync.WaitGroup, updateSite bool, knownScenes []string, out chan< // RegEx Patterns sceneIDRegEx := regexp.MustCompile(`^post-(\d+)`) dateRegEx := regexp.MustCompile(`(?i)^VideoPosted (?:on Premium )?on (.+)$`) - durationRegEx := regexp.MustCompile(`var timeAfter="(?:(\d+):)?(\d+):(\d+)";`) sceneCollector.OnHTML(`html`, func(e *colly.HTMLElement) { if !dateRegEx.MatchString(e.ChildText(`div.content-box.posted-by-box.posted-by-box-sub span.footer-titles`)) { @@ -134,16 +133,10 @@ func VRPorn(wg *sync.WaitGroup, updateSite bool, knownScenes []string, out chan< } // Duration - e.ForEachWithBreak(`script`, func(id int, e *colly.HTMLElement) bool { - var duration int - m := durationRegEx.FindStringSubmatch(e.Text) - if len(m) == 4 { - hours, _ := strconv.Atoi("0" + m[1]) - minutes, _ := strconv.Atoi(m[2]) - duration = hours*60 + minutes - } - sc.Duration = duration - return duration == 0 + e.ForEachWithBreak(`meta[property='og:duration']`, func(id int, e *colly.HTMLElement) bool { + secs, _ := strconv.Atoi(e.Attr("content")) + sc.Duration = secs / 60 + return sc.Duration == 0 }) out <- sc From f234eb39c143e791d1a326a48fff3aa8f429577b Mon Sep 17 00:00:00 2001 From: toshski <104477758+toshski@users.noreply.github.com> Date: Sat, 25 Nov 2023 01:10:52 +1300 Subject: [PATCH 27/33] scraper: Fix TmwVRnet scraper not picking up subsequent pages (#1510) --- pkg/scrape/tmwvrnet.go | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/pkg/scrape/tmwvrnet.go b/pkg/scrape/tmwvrnet.go index c53bf7b97..210c4a1dc 100644 --- a/pkg/scrape/tmwvrnet.go +++ b/pkg/scrape/tmwvrnet.go @@ -99,9 +99,11 @@ func TmwVRnet(wg *sync.WaitGroup, updateSite bool, knownScenes []string, out cha out <- sc }) - siteCollector.OnHTML(`div.pagination__element.next a`, func(e *colly.HTMLElement) { - pageURL := e.Request.AbsoluteURL(e.Attr("href")) - siteCollector.Visit(pageURL) + siteCollector.OnHTML(`a.pagination-element__link`, func(e *colly.HTMLElement) { + if strings.Contains(e.Text, "Next") { + pageURL := e.Request.AbsoluteURL(e.Attr("href")) + siteCollector.Visit(pageURL) + } }) siteCollector.OnHTML(`div.thumb-photo`, func(e *colly.HTMLElement) { From 7f6c3a8cd2803839b318fdacfe67ac4cc6d02754 Mon Sep 17 00:00:00 2001 From: crwxaj <52156245+crwxaj@users.noreply.github.com> Date: Fri, 24 Nov 2023 13:11:11 +0100 Subject: [PATCH 28/33] feat: add "Is Alpha Passthrough" attribute for SLR scenes, and pass it to HereSphere (#1500) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat: Add alpha passthrough marker to HereSphere API * add “Is Alpha Passthrough” attribute --------- Co-authored-by: crwxaj --- pkg/api/heresphere.go | 62 ++++++++++++++++++++++++--------------- pkg/api/scenes.go | 1 + pkg/models/model_scene.go | 2 ++ 3 files changed, 42 insertions(+), 23 deletions(-) diff --git a/pkg/api/heresphere.go b/pkg/api/heresphere.go index 14e56efe0..d45699fd2 100644 --- a/pkg/api/heresphere.go +++ b/pkg/api/heresphere.go @@ -18,6 +18,7 @@ import ( restfulspec "github.com/emicklei/go-restful-openapi/v2" "github.com/emicklei/go-restful/v3" "github.com/markphelps/optional" + "github.com/tidwall/gjson" "golang.org/x/crypto/bcrypt" "golang.org/x/text/language" "golang.org/x/text/language/display" @@ -38,29 +39,30 @@ type HeresphereListScenes struct { } type HeresphereVideo struct { - Access int `json:"access"` - Title string `json:"title"` - Description string `json:"description"` - ThumbnailImage string `json:"thumbnailImage"` - ThumbnailVideo string `json:"thumbnailVideo,omitempty"` - DateReleased string `json:"dateReleased"` - DateAdded string `json:"dateAdded"` - DurationMilliseconds uint `json:"duration"` - Rating float64 `json:"rating,omitempty"` - IsFavorite bool `json:"isFavorite"` - Projection string `json:"projection"` - Stereo string `json:"stereo"` - FOV float64 `json:"fov"` - Lens string `json:"lens"` - HspUrl string `json:"hsp,omitempty"` - Scripts []HeresphereScript `json:"scripts,omitempty"` - Subtitles []HeresphereSubtitles `json:"subtitles,omitempty"` - Tags []HeresphereTag `json:"tags,omitempty"` - Media []HeresphereMedia `json:"media"` - WriteFavorite bool `json:"writeFavorite"` - WriteRating bool `json:"writeRating"` - WriteTags bool `json:"writeTags"` - WriteHSP bool `json:"writeHSP"` + Access int `json:"access"` + Title string `json:"title"` + Description string `json:"description"` + ThumbnailImage string `json:"thumbnailImage"` + ThumbnailVideo string `json:"thumbnailVideo,omitempty"` + DateReleased string `json:"dateReleased"` + DateAdded string `json:"dateAdded"` + DurationMilliseconds uint `json:"duration"` + Rating float64 `json:"rating,omitempty"` + IsFavorite bool `json:"isFavorite"` + Projection string `json:"projection"` + Stereo string `json:"stereo"` + FOV float64 `json:"fov"` + Lens string `json:"lens"` + HspUrl string `json:"hsp,omitempty"` + Scripts []HeresphereScript `json:"scripts,omitempty"` + Subtitles []HeresphereSubtitles `json:"subtitles,omitempty"` + Tags []HeresphereTag `json:"tags,omitempty"` + Media []HeresphereMedia `json:"media"` + AlphaPackedSettings *HereSphereAlphaPackedSettings `json:"alphaPackedSettings,omitempty"` + WriteFavorite bool `json:"writeFavorite"` + WriteRating bool `json:"writeRating"` + WriteTags bool `json:"writeTags"` + WriteHSP bool `json:"writeHSP"` } type HeresphereScript struct { @@ -95,6 +97,10 @@ type HeresphereSource struct { URL string `json:"url"` } +type HereSphereAlphaPackedSettings struct { + DefaultSettings bool `json:"defaultSettings"` // not an actual setting, the struct does not get marshalled if it is empty +} + type HereSphereAuthRequest struct { Username string `json:"username"` Password string `json:"password"` @@ -696,6 +702,15 @@ func (i HeresphereResource) getHeresphereScene(req *restful.Request, resp *restf addFeatureTag("Year: " + scene.ReleaseDate.Format("2006")) } + var alphaPackedSettings *HereSphereAlphaPackedSettings = nil + if gjson.Valid(scene.ChromaKey) { + result := gjson.Get(scene.ChromaKey, "hasAlpha") + if result.Exists() && result.Bool() { + alphaPackedSettings = &HereSphereAlphaPackedSettings{DefaultSettings: true} + addFeatureTag("Is Alpha Passthrough") + } + } + for f := range features { tags = append(tags, HeresphereTag{ Name: "Feature:" + f, @@ -721,6 +736,7 @@ func (i HeresphereResource) getHeresphereScene(req *restful.Request, resp *restf Subtitles: heresphereSubtitlesFiles, Tags: tags, Media: media, + AlphaPackedSettings: alphaPackedSettings, WriteFavorite: config.Config.Interfaces.Heresphere.AllowFavoriteUpdates, WriteRating: config.Config.Interfaces.Heresphere.AllowRatingUpdates, WriteTags: config.Config.Interfaces.Heresphere.AllowTagUpdates || config.Config.Interfaces.Heresphere.AllowCuepointUpdates || config.Config.Interfaces.Heresphere.AllowWatchlistUpdates || config.Config.Web.SceneTrailerlist, diff --git a/pkg/api/scenes.go b/pkg/api/scenes.go index 313077354..474c2dc09 100644 --- a/pkg/api/scenes.go +++ b/pkg/api/scenes.go @@ -320,6 +320,7 @@ func (i SceneResource) getFilters(req *restful.Request, resp *restful.Response) outAttributes = append(outAttributes, "Is Favourite") outAttributes = append(outAttributes, "Is Scripted") outAttributes = append(outAttributes, "Is Passthrough") + outAttributes = append(outAttributes, "Is Alpha Passthrough") outAttributes = append(outAttributes, "In Watchlist") outAttributes = append(outAttributes, "In Wishlist") outAttributes = append(outAttributes, "Has Rating") diff --git a/pkg/models/model_scene.go b/pkg/models/model_scene.go index 980607413..c6cfa34ce 100644 --- a/pkg/models/model_scene.go +++ b/pkg/models/model_scene.go @@ -814,6 +814,8 @@ func queryScenes(db *gorm.DB, r RequestSceneList) (*gorm.DB, *gorm.DB) { where = "scenes.favourite = 1" case "Is Passthrough": where = "chroma_key <> ''" + case "Is Alpha Passthrough": + where = `chroma_key <> '' and chroma_key like '%"hasAlpha":true%'` case "In Wishlist": where = "wishlist = 1" case "Stashdb Linked": From 4f0bdb5069df09e5f4064473fdb7d24403104c23 Mon Sep 17 00:00:00 2001 From: toshski <104477758+toshski@users.noreply.github.com> Date: Sat, 25 Nov 2023 01:12:12 +1300 Subject: [PATCH 29/33] scraper: Fix trailers for VRLatina (#1506) --- pkg/api/trailers.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/api/trailers.go b/pkg/api/trailers.go index 8794b1e08..48fd8ae28 100644 --- a/pkg/api/trailers.go +++ b/pkg/api/trailers.go @@ -70,7 +70,7 @@ func ScrapeHtml(scrapeParams string) models.VideoSourceResponse { origURLtmp := e.Attr(params.ContentPath) quality := e.Attr(params.QualityPath) if origURLtmp != "" { - if params.ContentBaseUrl != "" { + if params.ContentBaseUrl != "" && !strings.HasPrefix(origURLtmp, params.ContentBaseUrl) { origURLtmp = params.ContentBaseUrl + origURLtmp } srcs = append(srcs, models.VideoSource{URL: origURLtmp, Quality: quality}) @@ -174,7 +174,7 @@ func extractFromJson(inputJson string, params models.TrailerScrape, srcs []model if params.EncodingPath != "" { encoding = gjson.Get(JsonMetadata, params.EncodingPath).String() + "-" } - if params.ContentBaseUrl != "" { + if params.ContentBaseUrl != "" && !strings.HasPrefix(url, params.ContentBaseUrl) { if params.ContentBaseUrl[len(params.ContentBaseUrl)-1:] == "/" && string(url[0]) == "/" { url = params.ContentBaseUrl + url[1:] } else { From f69e2c12ff0360c334f8c196edffefc827370d0e Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Fri, 24 Nov 2023 13:12:28 +0100 Subject: [PATCH 30/33] chore(deps): update dependency eslint to v8.54.0 (#1507) Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- package.json | 2 +- yarn.lock | 18 +++++++++--------- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/package.json b/package.json index 8185a6526..6f8c8ab2b 100644 --- a/package.json +++ b/package.json @@ -43,7 +43,7 @@ "@vue/eslint-config-standard": "8.0.1", "buffer": "6.0.3", "concurrently": "7.6.0", - "eslint": "8.53.0", + "eslint": "8.54.0", "eslint-plugin-import": "2.29.0", "eslint-plugin-node": "11.1.0", "eslint-plugin-promise": "6.1.1", diff --git a/yarn.lock b/yarn.lock index e943b4cd7..6988c204f 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1344,10 +1344,10 @@ minimatch "^3.1.2" strip-json-comments "^3.1.1" -"@eslint/js@8.53.0": - version "8.53.0" - resolved "https://registry.yarnpkg.com/@eslint/js/-/js-8.53.0.tgz#bea56f2ed2b5baea164348ff4d5a879f6f81f20d" - integrity sha512-Kn7K8dx/5U6+cT1yEhpX1w4PCSg0M+XyRILPgvwcEBjerFWCwQj5sbr3/VmxqV0JGHCBCzyd6LxypEuehypY1w== +"@eslint/js@8.54.0": + version "8.54.0" + resolved "https://registry.yarnpkg.com/@eslint/js/-/js-8.54.0.tgz#4fab9a2ff7860082c304f750e94acd644cf984cf" + integrity sha512-ut5V+D+fOoWPgGGNj83GGjnntO39xDy6DWxO0wb7Jp3DcMX0TfIqdzHF85VTQkerdyGmuuMD9AKAo5KiNlf/AQ== "@fortawesome/fontawesome-common-types@^0.2.36": version "0.2.36" @@ -4169,15 +4169,15 @@ eslint-webpack-plugin@^3.1.0: normalize-path "^3.0.0" schema-utils "^4.0.0" -eslint@8.53.0: - version "8.53.0" - resolved "https://registry.yarnpkg.com/eslint/-/eslint-8.53.0.tgz#14f2c8244298fcae1f46945459577413ba2697ce" - integrity sha512-N4VuiPjXDUa4xVeV/GC/RV3hQW9Nw+Y463lkWaKKXKYMvmRiRDAtfpuPFLN+E1/6ZhyR8J2ig+eVREnYgUsiag== +eslint@8.54.0: + version "8.54.0" + resolved "https://registry.yarnpkg.com/eslint/-/eslint-8.54.0.tgz#588e0dd4388af91a2e8fa37ea64924074c783537" + integrity sha512-NY0DfAkM8BIZDVl6PgSa1ttZbx3xHgJzSNJKYcQglem6CppHyMhRIQkBVSSMaSRnLhig3jsDbEzOjwCVt4AmmA== dependencies: "@eslint-community/eslint-utils" "^4.2.0" "@eslint-community/regexpp" "^4.6.1" "@eslint/eslintrc" "^2.1.3" - "@eslint/js" "8.53.0" + "@eslint/js" "8.54.0" "@humanwhocodes/config-array" "^0.11.13" "@humanwhocodes/module-importer" "^1.0.1" "@nodelib/fs.walk" "^1.2.8" From 16081de423a6c40412903ea7973cdbe0b449224c Mon Sep 17 00:00:00 2001 From: crwxaj <52156245+crwxaj@users.noreply.github.com> Date: Fri, 24 Nov 2023 16:14:03 +0100 Subject: [PATCH 31/33] scraper: add Czech AR scraper (#1512) Co-authored-by: crwxaj --- pkg/scrape/czechvr.go | 1 + 1 file changed, 1 insertion(+) diff --git a/pkg/scrape/czechvr.go b/pkg/scrape/czechvr.go index 24d1049d9..405c6da56 100644 --- a/pkg/scrape/czechvr.go +++ b/pkg/scrape/czechvr.go @@ -208,4 +208,5 @@ func init() { addCZVRScraper("czechvrfetish", "Czech VR Fetish", "16", "https://www.czechvrfetish.com/images/favicon/android-chrome-256x256.png") addCZVRScraper("czechvrcasting", "Czech VR Casting", "17", "https://www.czechvrcasting.com/images/favicon/android-chrome-256x256.png") addCZVRScraper("czechvrintimacy", "VR Intimacy", "18", "https://www.vrintimacy.com/images/favicon/android-chrome-256x256.png") + addCZVRScraper("czechar", "Czech AR", "19", "https://www.czechar.com/images/favicon/apple-touch-icon.png") } From 627fb834155e092f20b5ffb6843a8f9388f69cc5 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Fri, 24 Nov 2023 16:24:41 +0100 Subject: [PATCH 32/33] fix(deps): update module github.com/blevesearch/bleve_index_api to v1.1.3 (#1474) Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- go.mod | 2 +- go.sum | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/go.mod b/go.mod index 959c9b6b1..e7f111732 100644 --- a/go.mod +++ b/go.mod @@ -72,7 +72,7 @@ require ( github.com/antchfx/xpath v1.1.10 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/bits-and-blooms/bitset v1.2.0 // indirect - github.com/blevesearch/bleve_index_api v1.0.6 + github.com/blevesearch/bleve_index_api v1.1.3 github.com/blevesearch/geo v0.1.18 // indirect github.com/blevesearch/go-porterstemmer v1.0.3 // indirect github.com/blevesearch/gtreap v0.1.1 // indirect diff --git a/go.sum b/go.sum index 5c788e01b..c9aa8902b 100644 --- a/go.sum +++ b/go.sum @@ -44,6 +44,8 @@ github.com/blevesearch/bleve/v2 v2.3.10 h1:z8V0wwGoL4rp7nG/O3qVVLYxUqCbEwskMt4iR github.com/blevesearch/bleve/v2 v2.3.10/go.mod h1:RJzeoeHC+vNHsoLR54+crS1HmOWpnH87fL70HAUCzIA= github.com/blevesearch/bleve_index_api v1.0.6 h1:gyUUxdsrvmW3jVhhYdCVL6h9dCjNT/geNU7PxGn37p8= github.com/blevesearch/bleve_index_api v1.0.6/go.mod h1:YXMDwaXFFXwncRS8UobWs7nvo0DmusriM1nztTlj1ms= +github.com/blevesearch/bleve_index_api v1.1.3 h1:aNyMEiWFviY/1zYm7JCr2lZRIiYX0TMtz3oymxxbApc= +github.com/blevesearch/bleve_index_api v1.1.3/go.mod h1:PbcwjIcRmjhGbkS/lJCpfgVSMROV6TRubGGAODaK1W8= github.com/blevesearch/geo v0.1.18 h1:Np8jycHTZ5scFe7VEPLrDoHnnb9C4j636ue/CGrhtDw= github.com/blevesearch/geo v0.1.18/go.mod h1:uRMGWG0HJYfWfFJpK3zTdnnr1K+ksZTuWKhXeSokfnM= github.com/blevesearch/go-porterstemmer v1.0.3 h1:GtmsqID0aZdCSNiY8SkuPJ12pD4jI+DdXTAn4YRcHCo= From 174184feb209f687f183e2d951c10227069bf064 Mon Sep 17 00:00:00 2001 From: toshski <104477758+toshski@users.noreply.github.com> Date: Sat, 25 Nov 2023 05:26:00 +1300 Subject: [PATCH 33/33] feat: Add XBVR config settings to Backup/Restore (#1508) --- pkg/api/tasks.go | 3 ++- pkg/migrations/migrations.go | 2 +- pkg/tasks/content.go | 22 ++++++++++++++++++- .../sections/OptionsSceneDataImportExport.vue | 13 +++++++++-- 4 files changed, 35 insertions(+), 5 deletions(-) diff --git a/pkg/api/tasks.go b/pkg/api/tasks.go index 7d5a439c0..f1a090994 100644 --- a/pkg/api/tasks.go +++ b/pkg/api/tasks.go @@ -172,10 +172,11 @@ func (i TaskResource) backupBundle(req *restful.Request, resp *restful.Response) inclExtRefs, _ := strconv.ParseBool(req.QueryParameter("inclExtRefs")) inclActors, _ := strconv.ParseBool(req.QueryParameter("inclActors")) inclActorActions, _ := strconv.ParseBool(req.QueryParameter("inclActorActions")) + inclConfig, _ := strconv.ParseBool(req.QueryParameter("inclConfig")) playlistId := req.QueryParameter("playlistId") download := req.QueryParameter("download") - bundle := tasks.BackupBundle(inclAllSites, onlyIncludeOfficalSites, inclScenes, inclFileLinks, inclCuepoints, inclHistory, inclPlaylists, inclActorAkas, inclTagGroups, inclVolumes, inclSites, inclActions, inclExtRefs, inclActors, inclActorActions, playlistId, "", "") + bundle := tasks.BackupBundle(inclAllSites, onlyIncludeOfficalSites, inclScenes, inclFileLinks, inclCuepoints, inclHistory, inclPlaylists, inclActorAkas, inclTagGroups, inclVolumes, inclSites, inclActions, inclExtRefs, inclActors, inclActorActions, inclConfig, playlistId, "", "") if download == "true" { resp.WriteHeaderAndEntity(http.StatusOK, ResponseBackupBundle{Response: "Ready to Download from http://xxx.xxx.xxx.xxx:9999/download/xbvr-content-bundle.json"}) } else { diff --git a/pkg/migrations/migrations.go b/pkg/migrations/migrations.go index afa05ed4c..13c0647de 100644 --- a/pkg/migrations/migrations.go +++ b/pkg/migrations/migrations.go @@ -1483,7 +1483,7 @@ func Migrate() { } // backup bundle common.Log.Infof("Creating pre-migration backup, please waiit, backups can take some time on a system with a large number of scenes ") - tasks.BackupBundle(true, false, true, true, true, true, true, true, true, true, true, true, true, true, true, "0", "xbvr-premigration-bundle.json", "2") + tasks.BackupBundle(true, false, true, true, true, true, true, true, true, true, true, true, true, true, true, false, "0", "xbvr-premigration-bundle.json", "2") common.Log.Infof("Go to download/xbvr-premigration-bundle.json, or http://xxx.xxx.xxx.xxx:9999/download/xbvr-premigration-bundle.json if you need access to the backup") var sites []models.Site officalSiteChanges := []SiteChange{ diff --git a/pkg/tasks/content.go b/pkg/tasks/content.go index 7c0dc3df1..08495e36f 100644 --- a/pkg/tasks/content.go +++ b/pkg/tasks/content.go @@ -70,6 +70,7 @@ type BackupContentBundle struct { ExternalRefs []models.ExternalReference `xbvrbackup:"externalReferences"` Actors []models.Actor `xbvrbackup:"actors"` ActionActors []BackupActionActor `xbvrbackup:"actionActors"` + Kvs []models.KV `xbvrbackup:"config"` } type RequestRestore struct { InclAllSites bool `json:"allSites"` @@ -89,6 +90,7 @@ type RequestRestore struct { InclExternalRefs bool `json:"inclExtRefs"` InclActors bool `json:"inclActors"` InclActorActions bool `json:"inclActorActions"` + InclConfig bool `json:"inclConfig"` } func CleanTags() { @@ -512,7 +514,7 @@ func ImportBundleV1(bundleData ContentBundle) { } -func BackupBundle(inclAllSites bool, onlyIncludeOfficalSites bool, inclScenes bool, inclFileLinks bool, inclCuepoints bool, inclHistory bool, inclPlaylists bool, InclActorAkas bool, inclTagGroups bool, inclVolumes bool, inclSites bool, inclActions bool, inclExtRefs bool, inclActors bool, inclActorActions bool, playlistId string, outputBundleFilename string, version string) string { +func BackupBundle(inclAllSites bool, onlyIncludeOfficalSites bool, inclScenes bool, inclFileLinks bool, inclCuepoints bool, inclHistory bool, inclPlaylists bool, InclActorAkas bool, inclTagGroups bool, inclVolumes bool, inclSites bool, inclActions bool, inclExtRefs bool, inclActors bool, inclActorActions bool, inclConfig bool, playlistId string, outputBundleFilename string, version string) string { var out BackupContentBundle var content []byte exportCnt := 0 @@ -699,6 +701,10 @@ func BackupBundle(inclAllSites bool, onlyIncludeOfficalSites bool, inclScenes bo backupActionActorList = append(backupActionActorList, actorsActions) } } + var kvs []models.KV + if inclConfig { + db.Where("`key` not like 'lock%'").Find(&kvs) + } var err error out = BackupContentBundle{ @@ -717,6 +723,7 @@ func BackupBundle(inclAllSites bool, onlyIncludeOfficalSites bool, inclScenes bo ExternalRefs: externalReferences, Actors: actors, ActionActors: backupActionActorList, + Kvs: kvs, } var json = jsoniter.Config{ @@ -836,6 +843,9 @@ func RestoreBundle(request RequestRestore) { if request.InclActorActions { RestoreActionActors(bundleData.ActionActors, request.Overwrite, db) } + if request.InclConfig { + RestoreKvs(bundleData.Kvs, db) + } if request.InclScenes { CountTags() @@ -1542,6 +1552,16 @@ func RestoreActionActors(actionActorsList []BackupActionActor, overwrite bool, d } tlog.Infof("%v Actors with edits restored", addedCnt) } +func RestoreKvs(kvs []models.KV, db *gorm.DB) { + tlog := log.WithField("task", "scrape") + tlog.Infof("Restoring System Config") + + for _, kv := range kvs { + models.SaveWithRetry(db, &kv) + } + + tlog.Infof("System Config Restored ") +} func CountTags() { var tag models.Tag diff --git a/ui/src/views/options/sections/OptionsSceneDataImportExport.vue b/ui/src/views/options/sections/OptionsSceneDataImportExport.vue index d4a879f92..687a5f710 100644 --- a/ui/src/views/options/sections/OptionsSceneDataImportExport.vue +++ b/ui/src/views/options/sections/OptionsSceneDataImportExport.vue @@ -150,6 +150,13 @@ Include Scraper Settings + + + Include Config Settings + + Toggle Includes
@@ -219,6 +226,7 @@ export default { includePlaylists: true, includeVolumes: true, includeSites: true, + includeConfig: false, includeActorAkas: true, includeExternalReferences: true, includeTagGroups: true, @@ -300,13 +308,13 @@ export default { // put up a starting msg, as large files can cause it to appear to hang this.$store.state.messages.lastScrapeMessage = 'Starting restore' ky.post('/api/task/bundle/restore', { - json: { allSites: this.allSites == "true", onlyIncludeOfficalSites: this.onlyIncludeOfficalSites, inclScenes: this.includeScenes, inclHistory: this.includeHistory, inclLinks: this.includeFileLinks, inclCuepoints: this.includeCuepoints, inclActions: this.includeActions, inclPlaylists: this.includePlaylists, inclActorAkas: this.includeActorAkas, inclTagGroups: this.includeTagGroups, inclVolumes: this.includeVolumes, inclExtRefs: this.includeExternalReferences, inclSites: this.includeSites, inclActors: this.includeActors,inclActorActions: this.inclActorActions, overwrite: this.overwrite, uploadData: this.uploadData } + json: { allSites: this.allSites == "true", onlyIncludeOfficalSites: this.onlyIncludeOfficalSites, inclScenes: this.includeScenes, inclHistory: this.includeHistory, inclLinks: this.includeFileLinks, inclCuepoints: this.includeCuepoints, inclActions: this.includeActions, inclPlaylists: this.includePlaylists, inclActorAkas: this.includeActorAkas, inclTagGroups: this.includeTagGroups, inclVolumes: this.includeVolumes, inclExtRefs: this.includeExternalReferences, inclSites: this.includeSites, inclActors: this.includeActors,inclActorActions: this.inclActorActions, inclConfig: this.includeConfig, overwrite: this.overwrite, uploadData: this.uploadData } }) this.file = null } }, backupContent () { - ky.get('/api/task/bundle/backup', { timeout: false, searchParams: { allSites: this.allSites == "true", onlyIncludeOfficalSites: this.onlyIncludeOfficalSites, inclScenes: this.includeScenes, inclHistory: this.includeHistory, inclLinks: this.includeFileLinks, inclCuepoints: this.includeCuepoints, inclActions: this.includeActions, inclPlaylists: this.includePlaylists, inclActorAkas: this.includeActorAkas, inclTagGroups: this.includeTagGroups, inclVolumes: this.includeVolumes, inclExtRefs: this.includeExternalReferences, inclSites: this.includeSites, inclActors: this.includeActors,inclActorActions: this.inclActorActions, playlistId: this.currentPlaylist, download: true } }).json().then(data => { + ky.get('/api/task/bundle/backup', { timeout: false, searchParams: { allSites: this.allSites == "true", onlyIncludeOfficalSites: this.onlyIncludeOfficalSites, inclScenes: this.includeScenes, inclHistory: this.includeHistory, inclLinks: this.includeFileLinks, inclCuepoints: this.includeCuepoints, inclActions: this.includeActions, inclPlaylists: this.includePlaylists, inclActorAkas: this.includeActorAkas, inclTagGroups: this.includeTagGroups, inclVolumes: this.includeVolumes, inclExtRefs: this.includeExternalReferences, inclSites: this.includeSites, inclActors: this.includeActors,inclActorActions: this.inclActorActions, inclConfig: this.includeConfig, playlistId: this.currentPlaylist, download: true } }).json().then(data => { const link = document.createElement('a') link.href = this.myUrl link.click() @@ -330,6 +338,7 @@ export default { this.includePlaylists = !this.includePlaylists this.includeVolumes=!this.includeVolumes this.includeSites=!this.includeSites + this.includeConfig=!this.includeConfig }, } }