From 4990f4ccd44b786b66068138e477ae715dc62f42 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Wed, 24 Jan 2024 13:30:57 +0100 Subject: [PATCH 01/11] fix(deps): update dependency date-fns to v3.3.1 (#1600) Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- package.json | 2 +- yarn.lock | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/package.json b/package.json index e85fb126c..c2e334d5e 100644 --- a/package.json +++ b/package.json @@ -17,7 +17,7 @@ "@mdi/font": "7.4.47", "buefy": "0.9.27", "bulma-extensions": "6.2.7", - "date-fns": "3.2.0", + "date-fns": "3.3.1", "ky": "0.30.0", "pretty-bytes": "6.1.1", "videojs-hotkeys": "0.2.28", diff --git a/yarn.lock b/yarn.lock index 84c8ecf28..0e4fd752e 100644 --- a/yarn.lock +++ b/yarn.lock @@ -3469,10 +3469,10 @@ d@1, d@^1.0.1: es5-ext "^0.10.50" type "^1.0.1" -date-fns@3.2.0: - version "3.2.0" - resolved "https://registry.yarnpkg.com/date-fns/-/date-fns-3.2.0.tgz#c97cf685b62c829aa4ecba554e4a51768cf0bffc" - integrity sha512-E4KWKavANzeuusPi0jUjpuI22SURAznGkx7eZV+4i6x2A+IZxAMcajgkvuDAU1bg40+xuhW1zRdVIIM/4khuIg== +date-fns@3.3.1: + version "3.3.1" + resolved "https://registry.yarnpkg.com/date-fns/-/date-fns-3.3.1.tgz#7581daca0892d139736697717a168afbb908cfed" + integrity sha512-y8e109LYGgoQDveiEBD3DYXKba1jWf5BA8YU1FL5Tvm0BTdEfy54WLCwnuYWZNnzzvALy/QQ4Hov+Q9RVRv+Zw== date-fns@^2.29.1: version "2.29.3" From 0f7f2ec0b062629a93767b271744e3c2b130f6c9 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Wed, 24 Jan 2024 13:31:10 +0100 Subject: [PATCH 02/11] chore(deps): update dependency less-loader to v12.1.0 (#1598) Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- package.json | 2 +- yarn.lock | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/package.json b/package.json index c2e334d5e..ee9ad6f0a 100644 --- a/package.json +++ b/package.json @@ -50,7 +50,7 @@ "eslint-plugin-standard": "4.1.0", "eslint-plugin-vue": "9.20.1", "less": "4.2.0", - "less-loader": "12.0.0", + "less-loader": "12.1.0", "sass": "1.69.7", "sass-loader": "14.0.0", "simple-progress-webpack-plugin": "2.0.0", diff --git a/yarn.lock b/yarn.lock index 0e4fd752e..fe9c9fe57 100644 --- a/yarn.lock +++ b/yarn.lock @@ -5590,10 +5590,10 @@ launch-editor@^2.2.1: chalk "^2.3.0" shell-quote "^1.6.1" -less-loader@12.0.0: - version "12.0.0" - resolved "https://registry.yarnpkg.com/less-loader/-/less-loader-12.0.0.tgz#d45b322c01b3e6c3784298d7e4257ce25f714516" - integrity sha512-fcRoWK28+eD+1PxuwNG+44V2v32IBdzsYAi0keUncHVblbpxMPWwrGlnw0wZKCdOg7O0HNwfhWNw/DrRZ45xCA== +less-loader@12.1.0: + version "12.1.0" + resolved "https://registry.yarnpkg.com/less-loader/-/less-loader-12.1.0.tgz#d13878ed094c0fef87f12bb616c760eb5b366fca" + integrity sha512-N/MRZA9iILOW+TQ9xoDptsSPbtBJDWshOj3LNqL+UJAYDhtoraLECiBa93DeLJUfR4m/VE6bWuxaVs40+wBXYw== less@4.2.0: version "4.2.0" From 64ebcc7df1b3045e19f7e1ab51954797ae1eb936 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Wed, 24 Jan 2024 13:31:33 +0100 Subject: [PATCH 03/11] fix(deps): update module github.com/xo/dburl to v0.21.1 (#1593) Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- go.mod | 2 +- go.sum | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/go.mod b/go.mod index 8c8c3a880..4242377e2 100644 --- a/go.mod +++ b/go.mod @@ -54,7 +54,7 @@ require ( github.com/thoas/go-funk v0.9.3 github.com/tidwall/gjson v1.17.0 github.com/x-cray/logrus-prefixed-formatter v0.5.2 - github.com/xo/dburl v0.20.2 + github.com/xo/dburl v0.21.1 golang.org/x/crypto v0.18.0 golang.org/x/net v0.20.0 golang.org/x/oauth2 v0.16.0 diff --git a/go.sum b/go.sum index ac41fb775..17ab3b308 100644 --- a/go.sum +++ b/go.sum @@ -397,6 +397,8 @@ github.com/xo/dburl v0.18.3 h1:z271VmL/pk00rAF+0JrwrsOLyQcEBqqjyH3qX7eeJIE= github.com/xo/dburl v0.18.3/go.mod h1:B7/G9FGungw6ighV8xJNwWYQPMfn3gsi2sn5SE8Bzco= github.com/xo/dburl v0.20.2 h1:59zqIzahtfQ/X9E6fyp1ziHwjYEFy65opjpyQPmZC7w= github.com/xo/dburl v0.20.2/go.mod h1:B7/G9FGungw6ighV8xJNwWYQPMfn3gsi2sn5SE8Bzco= +github.com/xo/dburl v0.21.1 h1:n5mfH1fh51RQbvuaKKykGslodt8pZqyZJMNohVo2zK0= +github.com/xo/dburl v0.21.1/go.mod h1:B7/G9FGungw6ighV8xJNwWYQPMfn3gsi2sn5SE8Bzco= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= go.etcd.io/bbolt v1.3.7 h1:j+zJOnnEjF/kyHlDDgGnVL/AIqIJPq8UoB2GSNfkUfQ= go.etcd.io/bbolt v1.3.7/go.mod h1:N9Mkw9X8x5fupy0IKsmuqVtoGDyxsaDlbk4Rd05IAQw= From 6e83bb16a5aa97fcb6381f878a62e532c2b4c77a Mon Sep 17 00:00:00 2001 From: crwxaj <52156245+crwxaj@users.noreply.github.com> Date: Wed, 24 Jan 2024 13:57:42 +0100 Subject: [PATCH 04/11] scraper: Fix SLR covers (#1604) Co-authored-by: crwxaj --- pkg/scrape/slrstudios.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/scrape/slrstudios.go b/pkg/scrape/slrstudios.go index 659cf952a..f522b159c 100644 --- a/pkg/scrape/slrstudios.go +++ b/pkg/scrape/slrstudios.go @@ -67,7 +67,7 @@ func SexLikeReal(wg *sync.WaitGroup, updateSite bool, knownScenes []string, out if len(coverURL) > 0 { sc.Covers = append(sc.Covers, coverURL) } else { - m := coverRegEx.FindStringSubmatch(strings.TrimSpace(e.ChildAttr(`.splash-screen`, "style"))) + m := coverRegEx.FindStringSubmatch(strings.TrimSpace(e.ChildAttr(`.c-webxr-splash-screen`, "style"))) if len(m) > 0 && len(m[1]) > 0 { sc.Covers = append(sc.Covers, m[1]) } From 861bae5436b097d1515457a931fdd20cda0b84b8 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Wed, 24 Jan 2024 13:58:00 +0100 Subject: [PATCH 05/11] chore(deps): update dependency sass to v1.70.0 (#1594) Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- package.json | 2 +- yarn.lock | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/package.json b/package.json index ee9ad6f0a..c8fbf332e 100644 --- a/package.json +++ b/package.json @@ -51,7 +51,7 @@ "eslint-plugin-vue": "9.20.1", "less": "4.2.0", "less-loader": "12.1.0", - "sass": "1.69.7", + "sass": "1.70.0", "sass-loader": "14.0.0", "simple-progress-webpack-plugin": "2.0.0", "vue-cli-plugin-i18n": "2.3.2", diff --git a/yarn.lock b/yarn.lock index fe9c9fe57..83ae7ffd2 100644 --- a/yarn.lock +++ b/yarn.lock @@ -7241,10 +7241,10 @@ sass-loader@14.0.0: dependencies: neo-async "^2.6.2" -sass@1.69.7: - version "1.69.7" - resolved "https://registry.yarnpkg.com/sass/-/sass-1.69.7.tgz#6e7e1c8f51e8162faec3e9619babc7da780af3b7" - integrity sha512-rzj2soDeZ8wtE2egyLXgOOHQvaC2iosZrkF6v3EUG+tBwEvhqUCzm0VP3k9gHF9LXbSrRhT5SksoI56Iw8NPnQ== +sass@1.70.0: + version "1.70.0" + resolved "https://registry.yarnpkg.com/sass/-/sass-1.70.0.tgz#761197419d97b5358cb25f9dd38c176a8a270a75" + integrity sha512-uUxNQ3zAHeAx5nRFskBnrWzDUJrrvpCPD5FNAoRvTi0WwremlheES3tg+56PaVtCs5QDRX5CBLxxKMDJMEa1WQ== dependencies: chokidar ">=3.0.0 <4.0.0" immutable "^4.0.0" From b3fbe1b0deca29e299fc00532f1b4538e4ef792d Mon Sep 17 00:00:00 2001 From: theRealKLH <65736720+theRealKLH@users.noreply.github.com> Date: Wed, 24 Jan 2024 10:28:32 -0500 Subject: [PATCH 06/11] scraper: Fix duplicate actors on RealJamVR & PornCornVR (#1596) * Update realjamvr.go Fixed: Actors duped in newest 4 scenes due to tagging in Synopsis. * Update migrations.go added migration * Update migrations.go updated to handle existing PorncornVR scenes (up to 01/11/2024) --- pkg/migrations/migrations.go | 25 +++++++++++++++++++++++++ pkg/scrape/realjamvr.go | 2 +- 2 files changed, 26 insertions(+), 1 deletion(-) diff --git a/pkg/migrations/migrations.go b/pkg/migrations/migrations.go index 8bfdff604..06ba28da9 100644 --- a/pkg/migrations/migrations.go +++ b/pkg/migrations/migrations.go @@ -1860,6 +1860,31 @@ func Migrate() { return tx.Exec(sql).Error }, }, + { + ID: "0073-reset-RealJamVR-scenes-with-duped-actors", + Migrate: func(tx *gorm.DB) error { + + rjn := [...]string{"realjam-vr-39859", "realjam-vr-39861", "realjam-vr-40044", "realjam-vr-40064", "porncorn-vr-39827", "porncorn-vr-39902", "porncorn-vr-40031", "porncorn-vr-39903"} + var scenes []models.Scene + err := tx.Where("studio = ?", "Real Jam Network").Find(&scenes).Error + if err != nil { + return err + } + for _, scene := range scenes { + for _, v := range rjn { + if scene.SceneID == v { + scene.NeedsUpdate = true + err = tx.Save(&scene).Error + if err != nil { + return err + } + // common.Log.Infof("Updated scene %s", scene.SceneID) + } + } + } + return nil + }, + }, }) if err := m.Migrate(); err != nil { diff --git a/pkg/scrape/realjamvr.go b/pkg/scrape/realjamvr.go index b2cc6ac1f..913cc0b03 100644 --- a/pkg/scrape/realjamvr.go +++ b/pkg/scrape/realjamvr.go @@ -68,7 +68,7 @@ func RealJamSite(wg *sync.WaitGroup, updateSite bool, knownScenes []string, out // Cast sc.ActorDetails = make(map[string]models.ActorDetails) - e.ForEach(`div.scene-view a[href^='/actor/']`, func(id int, e *colly.HTMLElement) { + e.ForEach(`div.scene-view > a[href^='/actor/']`, func(id int, e *colly.HTMLElement) { sc.Cast = append(sc.Cast, strings.TrimSpace(e.Text)) sc.ActorDetails[strings.TrimSpace(e.Text)] = models.ActorDetails{Source: sc.ScraperID + " scrape", ProfileUrl: e.Request.AbsoluteURL(e.Attr("href"))} }) From 09db94b02c6ea62545475d86fb600aaea3f6f3b5 Mon Sep 17 00:00:00 2001 From: toshski <104477758+toshski@users.noreply.github.com> Date: Thu, 25 Jan 2024 04:29:02 +1300 Subject: [PATCH 07/11] Feat: File matching with stash hashs (#1587) * Lookup StashDB Ohash in File Matching * Create Option --- pkg/api/options.go | 29 +++++++++++++- pkg/config/config.go | 3 ++ pkg/scrape/stashdb.go | 17 +++++---- pkg/tasks/volume.go | 46 +++++++++++++++++++++++ ui/src/store/optionsStorage.js | 16 ++++++-- ui/src/views/options/sections/Storage.vue | 29 +++++++++++++- 6 files changed, 127 insertions(+), 13 deletions(-) diff --git a/pkg/api/options.go b/pkg/api/options.go index c00fa6497..a5aa0da51 100644 --- a/pkg/api/options.go +++ b/pkg/api/options.go @@ -178,6 +178,14 @@ type RequestSCustomSiteCreate struct { Company string `json:"scraperCompany"` } +type GetStorageResponse struct { + Volumes []models.Volume `json:"volumes"` + MatchOhash bool `json:"match_ohash"` +} +type RequestSaveOptionsStorage struct { + MatchOhash bool `json:"match_ohash"` +} + type ConfigResource struct{} func (i ConfigResource) WebService() *restful.WebService { @@ -228,6 +236,9 @@ func (i ConfigResource) WebService() *restful.WebService { Param(ws.PathParameter("storage-id", "Storage ID").DataType("int")). Metadata(restfulspec.KeyOpenAPITags, tags)) + ws.Route(ws.PUT("/storage").To(i.saveOptionsStorage). + Metadata(restfulspec.KeyOpenAPITags, tags)) + // "DLNA" section endpoints ws.Route(ws.PUT("/interface/dlna").To(i.saveOptionsDLNA). Metadata(restfulspec.KeyOpenAPITags, tags)) @@ -478,7 +489,10 @@ func (i ConfigResource) listStorage(req *restful.Request, resp *restful.Response (select sum(files.size) from files where files.volume_id = volumes.id) as total_size from volumes order by last_scan desc;`).Scan(&vol) - resp.WriteHeaderAndEntity(http.StatusOK, vol) + var out GetStorageResponse + out.Volumes = vol + out.MatchOhash = config.Config.Storage.MatchOhash + resp.WriteHeaderAndEntity(http.StatusOK, out) } func (i ConfigResource) addStorage(req *restful.Request, resp *restful.Response) { @@ -950,3 +964,16 @@ func (i ConfigResource) createCustomSite(req *restful.Request, resp *restful.Res resp.WriteHeader(http.StatusOK) } +func (i ConfigResource) saveOptionsStorage(req *restful.Request, resp *restful.Response) { + var r RequestSaveOptionsStorage + err := req.ReadEntity(&r) + if err != nil { + log.Error(err) + return + } + + config.Config.Storage.MatchOhash = r.MatchOhash + config.SaveConfig() + + resp.WriteHeaderAndEntity(http.StatusOK, r) +} diff --git a/pkg/config/config.go b/pkg/config/config.go index 1f0e7ddb6..baf6bf840 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -153,6 +153,9 @@ type ObjectConfig struct { RunAtStartDelay int `default:"0" json:"runAtStartDelay"` } `json:"stashdbRescrapeSchedule"` } `json:"cron"` + Storage struct { + MatchOhash bool `default:"false" json:"match_ohash"` + } `json:"storage"` } var ( diff --git a/pkg/scrape/stashdb.go b/pkg/scrape/stashdb.go index d9d73eb07..e0d59c7ad 100644 --- a/pkg/scrape/stashdb.go +++ b/pkg/scrape/stashdb.go @@ -145,7 +145,7 @@ func findStudio(studio string, field string) FindStudioResult { // Define the variables needed for your query as a Go map variables := `{"` + field + `": "` + studio + `"}` - resp := callStashDb(query, variables) + resp := CallStashDb(query, variables) var data FindStudioResult json.Unmarshal(resp, &data) return data @@ -191,7 +191,7 @@ func getPerformersPage(studioId string, page int) QueryPerformerResult { } ` - resp := callStashDb(query, variables) + resp := CallStashDb(query, variables) var data QueryPerformerResult json.Unmarshal(resp, &data) return data @@ -214,7 +214,7 @@ func getScenes(studioId string, parentId string, tagId string) QueryScenesResult } else { variables = getStudioSceneQueryVariable(studioId, page, count) } - sceneList = getScenePage(variables) + sceneList = GetScenePage(variables) nextList = sceneList for len(nextList.Data.QueryScenes.Scenes) > 0 && len(sceneList.Data.QueryScenes.Scenes) < sceneList.Data.QueryScenes.Count && // { @@ -225,7 +225,7 @@ func getScenes(studioId string, parentId string, tagId string) QueryScenesResult } else { variables = getStudioSceneQueryVariable(studioId, page, count) } - nextList = getScenePage(variables) + nextList = GetScenePage(variables) sceneList.Data.QueryScenes.Scenes = append(sceneList.Data.QueryScenes.Scenes, nextList.Data.QueryScenes.Scenes...) } return sceneList @@ -267,7 +267,7 @@ func getParentSceneQueryVariable(parentId string, tagId string, page int, count } // calls graphql scene query and return a list of scenes -func getScenePage(variables string) QueryScenesResult { +func GetScenePage(variables string) QueryScenesResult { query := ` query queryScenes($input: SceneQueryInput!) { queryScenes(input: $input) { @@ -325,7 +325,7 @@ func getScenePage(variables string) QueryScenesResult { ` // Define the variables needed for your query as a Go map - resp := callStashDb(query, variables) + resp := CallStashDb(query, variables) var data QueryScenesResult json.Unmarshal(resp, &data) return data @@ -492,14 +492,15 @@ func getStashPerformer(performer string) FindPerformerResult { // Define the variables needed for your query as a Go map var data FindPerformerResult variables := `{"id": "` + performer + `"}` - resp := callStashDb(query, variables) + resp := CallStashDb(query, variables) err := json.Unmarshal(resp, &data) if err != nil { log.Errorf("Eror extracting actor json") } return data } -func callStashDb(query string, rawVariables string) []byte { + +func CallStashDb(query string, rawVariables string) []byte { var variables map[string]interface{} json.Unmarshal([]byte(rawVariables), &variables) diff --git a/pkg/tasks/volume.go b/pkg/tasks/volume.go index b483cd453..c4a0d104b 100644 --- a/pkg/tasks/volume.go +++ b/pkg/tasks/volume.go @@ -19,8 +19,10 @@ import ( "github.com/sirupsen/logrus" "github.com/thoas/go-funk" "github.com/xbapps/xbvr/pkg/common" + "github.com/xbapps/xbvr/pkg/config" "github.com/xbapps/xbvr/pkg/ffprobe" "github.com/xbapps/xbvr/pkg/models" + "github.com/xbapps/xbvr/pkg/scrape" ) var allowedVideoExt = []string{".mp4", ".avi", ".wmv", ".mpeg4", ".mov", ".mkv"} @@ -84,6 +86,50 @@ func RescanVolumes(id int) { files[i].SceneID = scenes[0].ID files[i].Save() scenes[0].UpdateStatus() + } else { + if config.Config.Storage.MatchOhash && config.Config.Advanced.StashApiKey != "" { + hash := files[i].OsHash + if len(hash) < 16 { + // the has in xbvr is sometiomes < 16 pad with zeros + paddingLength := 16 - len(hash) + hash = strings.Repeat("0", paddingLength) + hash + } + queryVariable := ` + {"input":{ + "fingerprints": { + "value": "` + hash + `", + "modifier": "INCLUDES" + }, + "page": 1 + } + }` + // call Stashdb graphql searching for os_hash + stashMatches := scrape.GetScenePage(queryVariable) + for _, match := range stashMatches.Data.QueryScenes.Scenes { + if match.ID != "" { + var externalRefLink models.ExternalReferenceLink + db.Where(&models.ExternalReferenceLink{ExternalSource: "stashdb scene", ExternalId: match.ID}).First(&externalRefLink) + if externalRefLink.ID != 0 { + files[i].SceneID = externalRefLink.InternalDbId + files[i].Save() + var scene models.Scene + scene.GetIfExistByPK(externalRefLink.InternalDbId) + + // add filename tyo the array + var pfTxt []string + json.Unmarshal([]byte(scene.FilenamesArr), &pfTxt) + pfTxt = append(pfTxt, files[i].Filename) + tmp, _ := json.Marshal(pfTxt) + scene.FilenamesArr = string(tmp) + scene.Save() + models.AddAction(scene.SceneID, "match", "filenames_arr", scene.FilenamesArr) + + scene.UpdateStatus() + log.Infof("File %s matched to Scene %s matched using stashdb hash %s", path.Base(files[i].Filename), scene.SceneID, hash) + } + } + } + } } if (i % 50) == 0 { diff --git a/ui/src/store/optionsStorage.js b/ui/src/store/optionsStorage.js index b89f69ed6..d639b89c7 100644 --- a/ui/src/store/optionsStorage.js +++ b/ui/src/store/optionsStorage.js @@ -1,7 +1,10 @@ import ky from 'ky' const state = { - items: [] + items: [], + options: { + match_ohash: false, + }, } const mutations = { @@ -9,8 +12,15 @@ const mutations = { const actions = { async load ({ state }, params) { - state.items = await ky.get('/api/options/storage').json() - } + await ky.get('/api/options/storage').json() + .then(data => { + state.items = data.volumes + state.options.match_ohash = data.match_ohash + }) + }, + async save ({ state }, enabled) { + ky.put('/api/options/storage', { json: { ...state.options } }) + }, } export default { diff --git a/ui/src/views/options/sections/Storage.vue b/ui/src/views/options/sections/Storage.vue index e1feda518..db7d5981e 100644 --- a/ui/src/views/options/sections/Storage.vue +++ b/ui/src/views/options/sections/Storage.vue @@ -112,6 +112,22 @@ +
+ +
+

{{ $t('Options') }}

+ + + Match StashDB Hashes + + + + Save options + + + +
+ @@ -161,9 +177,20 @@ export default { }, rescanFolder: function (folder) { ky.get(`/api/task/rescan/${folder.id}`) - } + }, + save () { + this.$store.dispatch('optionsStorage/save') + }, }, computed: { + match_ohash: { + get () { + return this.$store.state.optionsStorage.options.match_ohash + }, + set (value) { + this.$store.state.optionsStorage.options.match_ohash = value + }, + }, total () { let files = 0; let unmatched = 0; let size = 0 this.$store.state.optionsStorage.items.map(v => { From 92c5193e4d693cd4908479e268fa6c24884ee895 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Wed, 24 Jan 2024 16:29:16 +0100 Subject: [PATCH 08/11] chore(deps): update dependency webpack to v5.90.0 (#1605) Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- package.json | 2 +- yarn.lock | 79 ++++++++++++++++++++++++++++++++++++---------------- 2 files changed, 56 insertions(+), 25 deletions(-) diff --git a/package.json b/package.json index c8fbf332e..f07267665 100644 --- a/package.json +++ b/package.json @@ -57,7 +57,7 @@ "vue-cli-plugin-i18n": "2.3.2", "vue-i18n-extract": "2.0.7", "vue-template-compiler": "2.7.16", - "webpack": "5.89.0" + "webpack": "5.90.0" }, "eslintConfig": { "root": true, diff --git a/yarn.lock b/yarn.lock index 83ae7ffd2..6111658fc 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1434,6 +1434,11 @@ resolved "https://registry.yarnpkg.com/@jridgewell/resolve-uri/-/resolve-uri-3.1.0.tgz#2203b118c157721addfe69d47b70465463066d78" integrity sha512-F2msla3tad+Mfht5cJq7LSXcdudKTWCVYUgw6pLFOOHSTtZlj6SWNYAp+AhuqLmWdBO2X5hPrLcu8cVP8fy28w== +"@jridgewell/resolve-uri@^3.1.0": + version "3.1.1" + resolved "https://registry.yarnpkg.com/@jridgewell/resolve-uri/-/resolve-uri-3.1.1.tgz#c08679063f279615a3326583ba3a90d1d82cc721" + integrity sha512-dSYZh7HhCDtCKm4QakX0xFpsRDqjjtZf/kjI/v3T3Nwt5r8/qz/M19F9ySyOqU94SXBmeG9ttTul+YnR4LOxFA== + "@jridgewell/set-array@^1.0.0", "@jridgewell/set-array@^1.0.1": version "1.1.2" resolved "https://registry.yarnpkg.com/@jridgewell/set-array/-/set-array-1.1.2.tgz#7c6cf998d6d20b914c0a55a91ae928ff25965e72" @@ -1447,11 +1452,24 @@ "@jridgewell/gen-mapping" "^0.3.0" "@jridgewell/trace-mapping" "^0.3.9" +"@jridgewell/source-map@^0.3.3": + version "0.3.5" + resolved "https://registry.yarnpkg.com/@jridgewell/source-map/-/source-map-0.3.5.tgz#a3bb4d5c6825aab0d281268f47f6ad5853431e91" + integrity sha512-UTYAUj/wviwdsMfzoSJspJxbkH5o1snzwX0//0ENX1u/55kkZZkcTZP6u9bwKGkv+dkk9at4m1Cpt0uY80kcpQ== + dependencies: + "@jridgewell/gen-mapping" "^0.3.0" + "@jridgewell/trace-mapping" "^0.3.9" + "@jridgewell/sourcemap-codec@1.4.14", "@jridgewell/sourcemap-codec@^1.4.10": version "1.4.14" resolved "https://registry.yarnpkg.com/@jridgewell/sourcemap-codec/-/sourcemap-codec-1.4.14.tgz#add4c98d341472a289190b424efbdb096991bb24" integrity sha512-XPSJHWmi394fuUuzDnGz1wiKqWfo1yXecHQMRf2l6hztTO+nPru658AyDngaBe7isIxEkRsPR3FZh+s7iVa4Uw== +"@jridgewell/sourcemap-codec@^1.4.14": + version "1.4.15" + resolved "https://registry.yarnpkg.com/@jridgewell/sourcemap-codec/-/sourcemap-codec-1.4.15.tgz#d7c6e6755c78567a951e04ab52ef0fd26de59f32" + integrity sha512-eF2rxCRulEKXHTRiDrDy6erMYWqNw4LPdQ8UQA4huuxaQsVeRPFl2oM8oDGxMFhJUWZf9McpLtJasDDZb/Bpeg== + "@jridgewell/trace-mapping@^0.3.17": version "0.3.18" resolved "https://registry.yarnpkg.com/@jridgewell/trace-mapping/-/trace-mapping-0.3.18.tgz#25783b2086daf6ff1dcb53c9249ae480e4dd4cd6" @@ -1460,6 +1478,14 @@ "@jridgewell/resolve-uri" "3.1.0" "@jridgewell/sourcemap-codec" "1.4.14" +"@jridgewell/trace-mapping@^0.3.20": + version "0.3.22" + resolved "https://registry.yarnpkg.com/@jridgewell/trace-mapping/-/trace-mapping-0.3.22.tgz#72a621e5de59f5f1ef792d0793a82ee20f645e4c" + integrity sha512-Wf963MzWtA2sjrNt+g18IAln9lKnlRp+K2eH4jjIoF1wYeq3aMREpG09xhlhdzS0EjwU7qmUJYangWa+151vZw== + dependencies: + "@jridgewell/resolve-uri" "^3.1.0" + "@jridgewell/sourcemap-codec" "^1.4.14" + "@jridgewell/trace-mapping@^0.3.7", "@jridgewell/trace-mapping@^0.3.9": version "0.3.14" resolved "https://registry.yarnpkg.com/@jridgewell/trace-mapping/-/trace-mapping-0.3.14.tgz#b231a081d8f66796e475ad588a1ef473112701ed" @@ -1611,10 +1637,10 @@ resolved "https://registry.yarnpkg.com/@types/estree/-/estree-0.0.51.tgz#cfd70924a25a3fd32b218e5e420e6897e1ac4f40" integrity sha512-CuPgU6f3eT/XgKKPqKd/gLZV1Xmvf1a2R5POBOGQa6uv82xpls89HU5zKeVoyR8XzHd1RGNOlQlvUe3CFkjWNQ== -"@types/estree@^1.0.0": - version "1.0.0" - resolved "https://registry.yarnpkg.com/@types/estree/-/estree-1.0.0.tgz#5fb2e536c1ae9bf35366eed879e827fa59ca41c2" - integrity sha512-WulqXMDUTYAXCjZnk6JtIHPigp55cVtDgDrO2gHRwhyJto21+1zbVCtOYB2L1F9w4qCQ0rOGWBnBe0FNTiEJIQ== +"@types/estree@^1.0.5": + version "1.0.5" + resolved "https://registry.yarnpkg.com/@types/estree/-/estree-1.0.5.tgz#a6ce3e556e00fd9895dd872dd172ad0d4bd687f4" + integrity sha512-/kYRxGDLWzHOB7q+wtSUQlFrtcdUccpfy+X+9iMBpHK8QLLhx2wIPYuS5DYtR9Wa/YlZAbIovy7qVdB1Aq6Lyw== "@types/express-serve-static-core@*", "@types/express-serve-static-core@^4.17.18": version "4.17.19" @@ -2396,6 +2422,11 @@ acorn@^8.0.4, acorn@^8.0.5, acorn@^8.5.0, acorn@^8.7.1: resolved "https://registry.yarnpkg.com/acorn/-/acorn-8.8.0.tgz#88c0187620435c7f6015803f5539dae05a9dbea8" integrity sha512-QOxyigPVrpZ2GXT+PFyZTl6TtOFc5egxHIP9IlQ+RbupQuX4RkT/Bee4/kQuC02Xkzg84JcT7oLYtDIQxp+v7w== +acorn@^8.8.2: + version "8.11.3" + resolved "https://registry.yarnpkg.com/acorn/-/acorn-8.11.3.tgz#71e0b14e13a4ec160724b38fb7b0f233b1b81d7a" + integrity sha512-Y9rRfJG5jcKOE0CLisYbojUjIrIEE7AGMzA/Sm4BslANhbS+cDMpgBdcPT91oJ7OuJ9hYJBx59RjbhxVnrF8Xg== + acorn@^8.9.0: version "8.9.0" resolved "https://registry.yarnpkg.com/acorn/-/acorn-8.9.0.tgz#78a16e3b2bcc198c10822786fa6679e245db5b59" @@ -2809,7 +2840,7 @@ browserslist@^4.0.0, browserslist@^4.14.5, browserslist@^4.16.3, browserslist@^4 node-releases "^2.0.5" update-browserslist-db "^1.0.4" -browserslist@^4.22.2: +browserslist@^4.21.10, browserslist@^4.22.2: version "4.22.2" resolved "https://registry.yarnpkg.com/browserslist/-/browserslist-4.22.2.tgz#704c4943072bd81ea18997f3bd2180e89c77874b" integrity sha512-0UgcrvQmBDvZHFGdYUehrCNIazki7/lUP3kkoi/r3YB2amZbFM9J43ZRkJTXBUZK4gmx56+Sqk9+Vs9mwZx9+A== @@ -7810,16 +7841,16 @@ terser-webpack-plugin@^5.1.1, terser-webpack-plugin@^5.1.3: serialize-javascript "^6.0.0" terser "^5.7.2" -terser-webpack-plugin@^5.3.7: - version "5.3.7" - resolved "https://registry.yarnpkg.com/terser-webpack-plugin/-/terser-webpack-plugin-5.3.7.tgz#ef760632d24991760f339fe9290deb936ad1ffc7" - integrity sha512-AfKwIktyP7Cu50xNjXF/6Qb5lBNzYaWpU6YfoX3uZicTx0zTy0stDDCsvjDapKsSDvOeWo5MEq4TmdBy2cNoHw== +terser-webpack-plugin@^5.3.10: + version "5.3.10" + resolved "https://registry.yarnpkg.com/terser-webpack-plugin/-/terser-webpack-plugin-5.3.10.tgz#904f4c9193c6fd2a03f693a2150c62a92f40d199" + integrity sha512-BKFPWlPDndPs+NGGCr1U59t0XScL5317Y0UReNrHaw9/FwhPENlq6bfgs+4yPfyP51vqC1bQ4rp1EfXW5ZSH9w== dependencies: - "@jridgewell/trace-mapping" "^0.3.17" + "@jridgewell/trace-mapping" "^0.3.20" jest-worker "^27.4.5" schema-utils "^3.1.1" serialize-javascript "^6.0.1" - terser "^5.16.5" + terser "^5.26.0" terser@^5.10.0, terser@^5.7.2: version "5.14.2" @@ -7831,13 +7862,13 @@ terser@^5.10.0, terser@^5.7.2: commander "^2.20.0" source-map-support "~0.5.20" -terser@^5.16.5: - version "5.16.9" - resolved "https://registry.yarnpkg.com/terser/-/terser-5.16.9.tgz#7a28cb178e330c484369886f2afd623d9847495f" - integrity sha512-HPa/FdTB9XGI2H1/keLFZHxl6WNvAI4YalHGtDQTlMnJcoqSab1UwL4l1hGEhs6/GmLHBZIg/YgB++jcbzoOEg== +terser@^5.26.0: + version "5.27.0" + resolved "https://registry.yarnpkg.com/terser/-/terser-5.27.0.tgz#70108689d9ab25fef61c4e93e808e9fd092bf20c" + integrity sha512-bi1HRwVRskAjheeYl291n3JC4GgO/Ty4z1nVs5AAsmonJulGxpSektecnNedrwK9C7vpvVtcX3cw00VSLt7U2A== dependencies: - "@jridgewell/source-map" "^0.3.2" - acorn "^8.5.0" + "@jridgewell/source-map" "^0.3.3" + acorn "^8.8.2" commander "^2.20.0" source-map-support "~0.5.20" @@ -8507,19 +8538,19 @@ webpack-virtual-modules@^0.4.2: resolved "https://registry.yarnpkg.com/webpack-virtual-modules/-/webpack-virtual-modules-0.4.4.tgz#a19fcf371923c59c4712d63d7d194b1e4d8262cc" integrity sha512-h9atBP/bsZohWpHnr+2sic8Iecb60GxftXsWNLLLSqewgIsGzByd2gcIID4nXcG+3tNe4GQG3dLcff3kXupdRA== -webpack@5.89.0: - version "5.89.0" - resolved "https://registry.yarnpkg.com/webpack/-/webpack-5.89.0.tgz#56b8bf9a34356e93a6625770006490bf3a7f32dc" - integrity sha512-qyfIC10pOr70V+jkmud8tMfajraGCZMBWJtrmuBymQKCrLTRejBI8STDp1MCyZu/QTdZSeacCQYpYNQVOzX5kw== +webpack@5.90.0: + version "5.90.0" + resolved "https://registry.yarnpkg.com/webpack/-/webpack-5.90.0.tgz#313bfe16080d8b2fee6e29b6c986c0714ad4290e" + integrity sha512-bdmyXRCXeeNIePv6R6tGPyy20aUobw4Zy8r0LUS2EWO+U+Ke/gYDgsCh7bl5rB6jPpr4r0SZa6dPxBxLooDT3w== dependencies: "@types/eslint-scope" "^3.7.3" - "@types/estree" "^1.0.0" + "@types/estree" "^1.0.5" "@webassemblyjs/ast" "^1.11.5" "@webassemblyjs/wasm-edit" "^1.11.5" "@webassemblyjs/wasm-parser" "^1.11.5" acorn "^8.7.1" acorn-import-assertions "^1.9.0" - browserslist "^4.14.5" + browserslist "^4.21.10" chrome-trace-event "^1.0.2" enhanced-resolve "^5.15.0" es-module-lexer "^1.2.1" @@ -8533,7 +8564,7 @@ webpack@5.89.0: neo-async "^2.6.2" schema-utils "^3.2.0" tapable "^2.1.1" - terser-webpack-plugin "^5.3.7" + terser-webpack-plugin "^5.3.10" watchpack "^2.4.0" webpack-sources "^3.2.3" From b80afdb72de1c756dca1754af3b18fd37973232a Mon Sep 17 00:00:00 2001 From: toshski <104477758+toshski@users.noreply.github.com> Date: Thu, 25 Jan 2024 04:37:59 +1300 Subject: [PATCH 09/11] Feat: Match Scenes to Alternate Sources (#1599) --- pkg/api/external_references.go | 128 +++++++- pkg/api/options.go | 101 ++++++- pkg/api/scenes.go | 72 +++++ pkg/api/tasks.go | 9 +- pkg/config/config.go | 28 +- pkg/config/scraper_list.go | 15 +- pkg/config/scrapers.json | 4 +- pkg/externalreference/stashdb.go | 1 - pkg/migrations/migrations.go | 28 +- pkg/models/model_external_reference.go | 35 ++- pkg/models/model_scene.go | 212 +++++++++---- pkg/models/model_scene_alternate_source.go | 109 +++++++ pkg/models/model_scraper.go | 15 +- pkg/models/model_site.go | 21 +- pkg/scrape/badoink.go | 4 +- pkg/scrape/czechvr.go | 3 +- pkg/scrape/povr.go | 25 +- pkg/scrape/scrape.go | 7 +- pkg/scrape/slrstudios.go | 43 ++- pkg/scrape/vrporn.go | 25 +- pkg/server/cron.go | 16 + pkg/tasks/alternate_scene_source.go | 280 ++++++++++++++++++ pkg/tasks/content.go | 139 ++++++--- pkg/tasks/volume.go | 27 +- ui/src/QuickFind.vue | 29 +- ui/src/components/EditButton.vue | 4 +- ui/src/locales/en-GB.json | 32 ++ ui/src/store/index.js | 4 +- ui/src/store/optionsAdvanced.js | 12 + ui/src/store/optionsSceneCreate.js | 25 ++ ui/src/store/overlay.js | 56 +++- ui/src/views/files/SceneMatch.vue | 3 +- ui/src/views/options/Options.vue | 23 +- .../options/overlays/SceneMatchParams.vue | 275 +++++++++++++++++ .../options/sections/InterfaceAdvanced.vue | 143 ++++++++- .../options/sections/OptionsSceneCreate.vue | 6 + .../sections/OptionsSceneDataImportExport.vue | 44 ++- .../sections/OptionsSceneDataScrapers.vue | 68 ++++- ui/src/views/options/sections/Schedules.vue | 72 ++++- ui/src/views/scenes/Details.vue | 237 +++++++++++++-- ui/src/views/scenes/EditScene.vue | 4 +- ui/src/views/scenes/Filters.vue | 1 + ui/src/views/scenes/SceneCard.vue | 41 ++- 43 files changed, 2180 insertions(+), 246 deletions(-) create mode 100644 pkg/models/model_scene_alternate_source.go create mode 100644 pkg/tasks/alternate_scene_source.go create mode 100644 ui/src/store/optionsSceneCreate.js create mode 100644 ui/src/views/options/overlays/SceneMatchParams.vue diff --git a/pkg/api/external_references.go b/pkg/api/external_references.go index 0666ec855..71ad0bd14 100644 --- a/pkg/api/external_references.go +++ b/pkg/api/external_references.go @@ -1,11 +1,26 @@ package api import ( + "net/http" + "time" + restfulspec "github.com/emicklei/go-restful-openapi/v2" "github.com/emicklei/go-restful/v3" + "github.com/xbapps/xbvr/pkg/models" ) -//var RequestBody []byte +// var RequestBody []byte +type RequestEditExtRefLink struct { + ID uint `json:"id"` + ExternalReferenceID uint `json:"external_reference_id"` + ExternalSource string `json:"external_source"` + ExternalId string `json:"external_id"` + MatchType int `json:"match_type"` + InternalTable string `json:"internal_table"` + InternalDbId uint `json:"internal_db_id"` + InternalNameId string `json:"internal_name_id"` + DeleteDate time.Time `json:"delete_date"` +} type ExternalReference struct{} @@ -39,5 +54,116 @@ func (i ExternalReference) WebService() *restful.WebService { Metadata(restfulspec.KeyOpenAPITags, tags)) ws.Route(ws.GET("/generic/scrape_by_site/{site-id}").To(i.genericActorScraperBySite). Metadata(restfulspec.KeyOpenAPITags, tags)) + ws.Route(ws.POST("/edit_link").To(i.editExtRefLink). + Metadata(restfulspec.KeyOpenAPITags, tags). + Writes()) + ws.Route(ws.DELETE("/delete_extref").To(i.deleteExtRefLink). + Metadata(restfulspec.KeyOpenAPITags, tags). + Writes()) + ws.Route(ws.DELETE("/delete_extref_source").To(i.deleteExtRefSource). + Metadata(restfulspec.KeyOpenAPITags, tags). + Writes()) + ws.Route(ws.DELETE("/delete_extref_source_links/all").To(i.deleteExtRefSourceLinks). + Metadata(restfulspec.KeyOpenAPITags, tags). + Writes()) + ws.Route(ws.DELETE("/delete_extref_source_links/keep_manual").To(i.deleteExtRefSourceLinksKeepManualMatches). + Metadata(restfulspec.KeyOpenAPITags, tags). + Writes()) return ws } + +func (i ExternalReference) editExtRefLink(req *restful.Request, resp *restful.Response) { + var r RequestEditExtRefLink + err := req.ReadEntity(&r) + if err != nil { + log.Error(err) + return + } + + var extreflink models.ExternalReferenceLink + if r.ID > 0 { + extreflink.ExternalReference.GetIfExist(r.ID) + } else { + extreflink.FindByExternaID(r.ExternalSource, r.ExternalId) + } + extreflink.InternalTable = r.InternalTable + extreflink.InternalDbId = r.InternalDbId + extreflink.InternalNameId = r.InternalNameId + extreflink.MatchType = r.MatchType + extreflink.Save() + resp.WriteHeaderAndEntity(http.StatusOK, extreflink) +} +func (i ExternalReference) deleteExtRefLink(req *restful.Request, resp *restful.Response) { + // delete a single external_reference_link + var r RequestEditExtRefLink + err := req.ReadEntity(&r) + if err != nil { + log.Error(err) + return + } + + var extreflink models.ExternalReferenceLink + if r.ID > 0 { + extreflink.ExternalReference.GetIfExist(r.ID) + } else { + extreflink.FindByExternaID(r.ExternalSource, r.ExternalId) + } + extreflink.ExternalReference.Delete() + extreflink.Delete() + resp.WriteHeaderAndEntity(http.StatusOK, nil) +} +func (i ExternalReference) deleteExtRefSource(req *restful.Request, resp *restful.Response) { + // deletes all external_reference_links and external_references for a source + var r RequestEditExtRefLink + err := req.ReadEntity(&r) + if err != nil { + log.Error(err) + return + } + commonDb, _ := models.GetCommonDB() + + commonDb.Where("external_source = ?", r.ExternalSource).Delete(models.ExternalReferenceLink{}) + commonDb.Where("external_source = ?", r.ExternalSource).Delete(models.ExternalReference{}) + + resp.WriteHeaderAndEntity(http.StatusOK, nil) +} +func (i ExternalReference) deleteExtRefSourceLinks(req *restful.Request, resp *restful.Response) { + // deletes external_reference_links for a source + var r RequestEditExtRefLink + err := req.ReadEntity(&r) + if err != nil { + log.Error(err) + return + } + commonDb, _ := models.GetCommonDB() + commonDb.Where("external_source like ?", r.ExternalSource).Delete(models.ExternalReferenceLink{}) + + resp.WriteHeaderAndEntity(http.StatusOK, nil) +} +func (i ExternalReference) deleteExtRefSourceLinksKeepManualMatches(req *restful.Request, resp *restful.Response) { + // deletes external_reference_links for a source, but keeps links the user has manually set, ie match_type = 99999 + var r RequestEditExtRefLink + err := req.ReadEntity(&r) + if err != nil { + log.Error(err) + return + } + db, _ := models.GetDB() + defer db.Close() + + if r.DeleteDate.IsZero() { + db.Where("external_source like ? and match_type not in (99999, -1)", r.ExternalSource).Delete(models.ExternalReferenceLink{}) + } else { + // Fetch records to delete + var recordsToDelete []models.ExternalReferenceLink + db.Debug().Joins("JOIN external_references ON external_reference_links.external_reference_id = external_references.id"). + Where("external_reference_links.external_source LIKE ? AND match_type NOT IN (99999, -1) AND external_references.external_date >= ?", r.ExternalSource, r.DeleteDate). + Find(&recordsToDelete) + for _, record := range recordsToDelete { + db.Debug().Delete(&record) + } + + } + + resp.WriteHeaderAndEntity(http.StatusOK, nil) +} diff --git a/pkg/api/options.go b/pkg/api/options.go index a5aa0da51..0e918ed3d 100644 --- a/pkg/api/options.go +++ b/pkg/api/options.go @@ -62,12 +62,16 @@ type RequestSaveOptionsWeb struct { } type RequestSaveOptionsAdvanced struct { - ShowInternalSceneId bool `json:"showInternalSceneId"` - ShowHSPApiLink bool `json:"showHSPApiLink"` - ShowSceneSearchField bool `json:"showSceneSearchField"` - StashApiKey string `json:"stashApiKey"` - ScrapeActorAfterScene bool `json:"scrapeActorAfterScene"` - UseImperialEntry bool `json:"useImperialEntry"` + ShowInternalSceneId bool `json:"showInternalSceneId"` + ShowHSPApiLink bool `json:"showHSPApiLink"` + ShowSceneSearchField bool `json:"showSceneSearchField"` + StashApiKey string `json:"stashApiKey"` + ScrapeActorAfterScene bool `json:"scrapeActorAfterScene"` + UseImperialEntry bool `json:"useImperialEntry"` + LinkScenesAfterSceneScraping bool `json:"linkScenesAfterSceneScraping"` + UseAltSrcInFileMatching bool `json:"useAltSrcInFileMatching"` + UseAltSrcInScriptFilters bool `json:"useAltSrcInScriptFilters"` + IgnoreReleasedBefore time.Time `json:"ignoreReleasedBefore"` } type RequestSaveOptionsFunscripts struct { @@ -165,6 +169,18 @@ type RequestSaveOptionsTaskSchedule struct { StashdbRescrapeHourStart int `json:"stashdbRescrapeHourStart"` StashdbRescrapeHourEnd int `json:"stashdbRescrapeHourEnd"` StashdbRescrapeStartDelay int `json:"stashdbRescrapeStartDelay"` + + LinkScenesEnabled bool `json:"linkScenesEnabled"` + LinkScenesHourInterval int `json:"linkScenesHourInterval"` + LinkScenesUseRange bool `json:"linkScenesUseRange"` + LinkScenesMinuteStart int `json:"linkScenesMinuteStart"` + LinkScenesHourStart int `json:"linkScenesHourStart"` + LinkScenesHourEnd int `json:"linkScenesHourEnd"` + LinkScenesStartDelay int `json:"linkScenesStartDelay"` +} +type RequestSaveSiteMatchParams struct { + SiteId string `json:"site"` + MatchParams models.AltSrcMatchParams `json:"match_params"` } type RequestCuepointsResponse struct { @@ -172,10 +188,11 @@ type RequestCuepointsResponse struct { Actions []string `json:"actions"` } type RequestSCustomSiteCreate struct { - Url string `json:"scraperUrl"` - Name string `json:"scraperName"` - Avatar string `json:"scraperAvatar"` - Company string `json:"scraperCompany"` + Url string `json:"scraperUrl"` + Name string `json:"scraperName"` + Avatar string `json:"scraperAvatar"` + Company string `json:"scraperCompany"` + MasterSiteId string `json:"masterSiteId"` } type GetStorageResponse struct { @@ -225,6 +242,11 @@ func (i ConfigResource) WebService() *restful.WebService { ws.Route(ws.POST("/scraper/delete-scenes").To(i.deleteScenes). Metadata(restfulspec.KeyOpenAPITags, tags)) + ws.Route(ws.GET("/site/match_params/{site}").To(i.siteMatchParams). + Metadata(restfulspec.KeyOpenAPITags, tags)) + ws.Route(ws.POST("/site/save_match_params").To(i.saveSiteMatchParams). + Metadata(restfulspec.KeyOpenAPITags, tags)) + // "Storage" section endpoints ws.Route(ws.GET("/storage").To(i.listStorage). Metadata(restfulspec.KeyOpenAPITags, tags)) @@ -382,6 +404,49 @@ func (i ConfigResource) listSitesWithDB(req *restful.Request, resp *restful.Resp resp.WriteHeaderAndEntity(http.StatusOK, sites) } +func (i ConfigResource) siteMatchParams(req *restful.Request, resp *restful.Response) { + db, _ := models.GetDB() + defer db.Close() + + id := req.PathParameter("site") + if id == "" { + return + } + + var site models.Site + err := site.GetIfExist(id) + if err != nil { + log.Error(err) + return + } + + var matchParams models.AltSrcMatchParams + matchParams.UnmarshalParams(site.MatchingParams) + resp.WriteHeaderAndEntity(http.StatusOK, matchParams) +} +func (i ConfigResource) saveSiteMatchParams(req *restful.Request, resp *restful.Response) { + db, _ := models.GetDB() + defer db.Close() + var r RequestSaveSiteMatchParams + if err := req.ReadEntity(&r); err != nil { + APIError(req, resp, http.StatusInternalServerError, err) + return + } + + var site models.Site + err := site.GetIfExist(r.SiteId) + if err != nil { + log.Error(err) + return + } + + json, _ := json.Marshal(r.MatchParams) + site.MatchingParams = string(json) + site.Save() + + resp.WriteHeaderAndEntity(http.StatusOK, nil) +} + func (i ConfigResource) saveOptionsWeb(req *restful.Request, resp *restful.Response) { var r RequestSaveOptionsWeb err := req.ReadEntity(&r) @@ -425,6 +490,10 @@ func (i ConfigResource) saveOptionsAdvanced(req *restful.Request, resp *restful. config.Config.Advanced.StashApiKey = r.StashApiKey config.Config.Advanced.ScrapeActorAfterScene = r.ScrapeActorAfterScene config.Config.Advanced.UseImperialEntry = r.UseImperialEntry + config.Config.Advanced.LinkScenesAfterSceneScraping = r.LinkScenesAfterSceneScraping + config.Config.Advanced.UseAltSrcInFileMatching = r.UseAltSrcInFileMatching + config.Config.Advanced.UseAltSrcInScriptFilters = r.UseAltSrcInScriptFilters + config.Config.Advanced.IgnoreReleasedBefore = r.IgnoreReleasedBefore config.SaveConfig() resp.WriteHeaderAndEntity(http.StatusOK, r) @@ -873,6 +942,14 @@ func (i ConfigResource) saveOptionsTaskSchedule(req *restful.Request, resp *rest config.Config.Cron.StashdbRescrapeSchedule.HourEnd = r.StashdbRescrapeHourEnd config.Config.Cron.StashdbRescrapeSchedule.RunAtStartDelay = r.StashdbRescrapeStartDelay + config.Config.Cron.LinkScenesSchedule.Enabled = r.LinkScenesEnabled + config.Config.Cron.LinkScenesSchedule.HourInterval = r.LinkScenesHourInterval + config.Config.Cron.LinkScenesSchedule.UseRange = r.LinkScenesUseRange + config.Config.Cron.LinkScenesSchedule.MinuteStart = r.LinkScenesMinuteStart + config.Config.Cron.LinkScenesSchedule.HourStart = r.LinkScenesHourStart + config.Config.Cron.LinkScenesSchedule.HourEnd = r.LinkScenesHourEnd + config.Config.Cron.LinkScenesSchedule.RunAtStartDelay = r.LinkScenesStartDelay + config.SaveConfig() resp.WriteHeaderAndEntity(http.StatusOK, r) @@ -908,6 +985,7 @@ func (i ConfigResource) createCustomSite(req *restful.Request, resp *restful.Res r.Name = strings.TrimSpace(r.Name) r.Company = strings.TrimSpace(r.Company) r.Avatar = strings.TrimSpace(r.Avatar) + r.MasterSiteId = strings.TrimSpace(r.MasterSiteId) if r.Company == "" { r.Company = r.Name } @@ -937,12 +1015,13 @@ func (i ConfigResource) createCustomSite(req *restful.Request, resp *restful.Res scrapers[key][idx].Name = r.Name scrapers[key][idx].Company = r.Company scrapers[key][idx].AvatarUrl = r.Avatar + scrapers[key][idx].MasterSiteId = r.MasterSiteId } } } if !exists { - scraper := config.ScraperConfig{URL: r.Url, Name: r.Name, Company: r.Company, AvatarUrl: r.Avatar} + scraper := config.ScraperConfig{URL: r.Url, Name: r.Name, Company: r.Company, AvatarUrl: r.Avatar, MasterSiteId: r.MasterSiteId} switch match[3] { case "povr": scrapers["povr"] = append(scrapers["povr"], scraper) diff --git a/pkg/api/scenes.go b/pkg/api/scenes.go index 25edba5c0..4b20e3047 100644 --- a/pkg/api/scenes.go +++ b/pkg/api/scenes.go @@ -1,6 +1,7 @@ package api import ( + "encoding/json" "fmt" "net/http" "strconv" @@ -86,6 +87,14 @@ type ResponseSceneSearchValue struct { FieldName string `json:"fieldName"` FieldValue string `json:"fieldValue"` } +type ResponseGetAlternateSources struct { + Url string `json:"url"` + Icon string `json:"site_icon"` + ExternalSource string `json:"external_source"` + ExternalId string `json:"external_id"` + ExternalData string `json:"external_data"` +} + type SceneResource struct{} func (i SceneResource) WebService() *restful.WebService { @@ -148,6 +157,10 @@ func (i SceneResource) WebService() *restful.WebService { Metadata(restfulspec.KeyOpenAPITags, tags). Writes(models.Scene{})) + ws.Route(ws.GET("/alternate_source/{scene-id}").To(i.getSceneAlternateSources). + Metadata(restfulspec.KeyOpenAPITags, tags). + Writes(ResponseGetAlternateSources{})) + return ws } @@ -373,6 +386,10 @@ func (i SceneResource) getFilters(req *restful.Request, resp *restful.Response) outAttributes = append(outAttributes, "Has AI Generated Script") outAttributes = append(outAttributes, "Has Human Generated Script") outAttributes = append(outAttributes, "Has Favourite Actor") + outAttributes = append(outAttributes, "Available from Alternate Sites") + outAttributes = append(outAttributes, "Available from POVR") + outAttributes = append(outAttributes, "Available from VRPorn") + outAttributes = append(outAttributes, "Available from SLR") type Results struct { Result string } @@ -598,6 +615,28 @@ func (i SceneResource) searchSceneIndex(req *restful.Request, resp *restful.Resp log.Error(err) return } + if strings.HasPrefix(q, "http") { + // if searching for a link, see if it is in the external ref table for scene alternate source + var extref models.ExternalReference + var scene models.Scene + splits := strings.Split(q, "?") + q = splits[0] + + // see if the url matches a scrapped scene + scene.GetIfExistURL(q) + if scene.ID != 0 { + scenes = append(scenes, scene) + } else { + db.Preload("XbvrLinks").Where("(external_source like 'alternate scene %' or external_source = 'stashdb scene') and external_url = ?", q).First(&extref) + for _, link := range extref.XbvrLinks { + if link.InternalTable == "scenes" { + scene.GetIfExistByPK(link.InternalDbId) + scenes = append(scenes, scene) + } + } + } + } + defer idx.Bleve.Close() query := bleve.NewQueryStringQuery(q) @@ -938,3 +977,36 @@ func ProcessTagChanges(scene *models.Scene, tags *[]string, db *gorm.DB) { } } } +func (i SceneResource) getSceneAlternateSources(req *restful.Request, resp *restful.Response) { + var extref models.ExternalReferenceLink + var refs []models.ExternalReferenceLink + var ressults []ResponseGetAlternateSources + db, _ := models.GetDB() + + if strings.Contains(req.PathParameter("scene-id"), "-") { + refs = extref.FindByInternalName("scenes", req.PathParameter("scene-id")) + } else { + id, err := strconv.Atoi(req.PathParameter("scene-id")) + if err != nil { + log.Error(err) + return + } + refs = extref.FindByInternalID("scenes", uint(id)) + } + + for _, ref := range refs { + var altscene models.SceneAlternateSource + var site models.Site + + if ref.ExternalSource == "stashdb scene" { + ressults = append(ressults, ResponseGetAlternateSources{Url: ref.ExternalReference.ExternalURL, Icon: "https://docs.stashapp.cc/favicon.ico", ExternalSource: ref.ExternalReference.ExternalSource, ExternalId: ref.ExternalReference.ExternalId, ExternalData: ref.ExternalReference.ExternalData}) + } else { + json.Unmarshal([]byte(ref.ExternalReference.ExternalData), &altscene) + site.GetIfExist(altscene.Scene.ScraperId) + ressults = append(ressults, ResponseGetAlternateSources{Url: ref.ExternalReference.ExternalURL, Icon: site.AvatarURL, ExternalSource: ref.ExternalReference.ExternalSource, ExternalId: ref.ExternalReference.ExternalId, ExternalData: ref.ExternalReference.ExternalData}) + } + } + db.Close() + + resp.WriteHeaderAndEntity(http.StatusOK, ressults) +} diff --git a/pkg/api/tasks.go b/pkg/api/tasks.go index f1a090994..f3a119b56 100644 --- a/pkg/api/tasks.go +++ b/pkg/api/tasks.go @@ -100,6 +100,8 @@ func (i TaskResource) WebService() *restful.WebService { ws.Route(ws.POST("/scrape-tpdb").To(i.scrapeTPDB). Metadata(restfulspec.KeyOpenAPITags, tags)) + ws.Route(ws.GET("/relink_alt_aource_scenes").To(i.relink_alt_aource_scenes). + Metadata(restfulspec.KeyOpenAPITags, tags)) return ws } @@ -173,10 +175,12 @@ func (i TaskResource) backupBundle(req *restful.Request, resp *restful.Response) inclActors, _ := strconv.ParseBool(req.QueryParameter("inclActors")) inclActorActions, _ := strconv.ParseBool(req.QueryParameter("inclActorActions")) inclConfig, _ := strconv.ParseBool(req.QueryParameter("inclConfig")) + extRefSubset := req.QueryParameter("extRefSubset") playlistId := req.QueryParameter("playlistId") download := req.QueryParameter("download") - bundle := tasks.BackupBundle(inclAllSites, onlyIncludeOfficalSites, inclScenes, inclFileLinks, inclCuepoints, inclHistory, inclPlaylists, inclActorAkas, inclTagGroups, inclVolumes, inclSites, inclActions, inclExtRefs, inclActors, inclActorActions, inclConfig, playlistId, "", "") + bundle := tasks.BackupBundle(inclAllSites, onlyIncludeOfficalSites, inclScenes, inclFileLinks, inclCuepoints, inclHistory, inclPlaylists, + inclActorAkas, inclTagGroups, inclVolumes, inclSites, inclActions, inclExtRefs, inclActors, inclActorActions, inclConfig, extRefSubset, playlistId, "", "") if download == "true" { resp.WriteHeaderAndEntity(http.StatusOK, ResponseBackupBundle{Response: "Ready to Download from http://xxx.xxx.xxx.xxx:9999/download/xbvr-content-bundle.json"}) } else { @@ -226,3 +230,6 @@ func (i TaskResource) scrapeTPDB(req *restful.Request, resp *restful.Response) { go tasks.ScrapeTPDB(strings.TrimSpace(r.ApiToken), strings.TrimSpace(r.SceneUrl)) } } +func (i TaskResource) relink_alt_aource_scenes(req *restful.Request, resp *restful.Response) { + go tasks.MatchAlternateSources() +} diff --git a/pkg/config/config.go b/pkg/config/config.go index baf6bf840..257024a4f 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -2,6 +2,7 @@ package config import ( "encoding/json" + "time" "github.com/creasty/defaults" @@ -47,13 +48,17 @@ type ObjectConfig struct { IsAvailOpacity int `default:"40" json:"isAvailOpacity"` } `json:"web"` Advanced struct { - ShowInternalSceneId bool `default:"false" json:"showInternalSceneId"` - ShowHSPApiLink bool `default:"false" json:"showHSPApiLink"` - ShowSceneSearchField bool `default:"false" json:"showSceneSearchField"` - StashApiKey string `default:"" json:"stashApiKey"` - ScrapeActorAfterScene bool `default:"true" json:"scrapeActorAfterScene"` - UseImperialEntry bool `default:"false" json:"useImperialEntry"` - ProgressTimeInterval int `default:"15" json:"progressTimeInterval"` + ShowInternalSceneId bool `default:"false" json:"showInternalSceneId"` + ShowHSPApiLink bool `default:"false" json:"showHSPApiLink"` + ShowSceneSearchField bool `default:"false" json:"showSceneSearchField"` + StashApiKey string `default:"" json:"stashApiKey"` + ScrapeActorAfterScene bool `default:"true" json:"scrapeActorAfterScene"` + UseImperialEntry bool `default:"false" json:"useImperialEntry"` + ProgressTimeInterval int `default:"15" json:"progressTimeInterval"` + LinkScenesAfterSceneScraping bool `default:"true" json:"linkScenesAfterSceneScraping"` + UseAltSrcInFileMatching bool `default:"true" json:"useAltSrcInFileMatching"` + UseAltSrcInScriptFilters bool `default:"true" json:"useAltSrcInScriptFilters"` + IgnoreReleasedBefore time.Time `json:"ignoreReleasedBefore"` } `json:"advanced"` Funscripts struct { ScrapeFunscripts bool `default:"false" json:"scrapeFunscripts"` @@ -152,6 +157,15 @@ type ObjectConfig struct { HourEnd int `default:"23" json:"hourEnd"` RunAtStartDelay int `default:"0" json:"runAtStartDelay"` } `json:"stashdbRescrapeSchedule"` + LinkScenesSchedule struct { + Enabled bool `default:"false" json:"enabled"` + HourInterval int `default:"12" json:"hourInterval"` + UseRange bool `default:"false" json:"useRange"` + MinuteStart int `default:"0" json:"minuteStart"` + HourStart int `default:"0" json:"hourStart"` + HourEnd int `default:"23" json:"hourEnd"` + RunAtStartDelay int `default:"0" json:"runAtStartDelay"` + } `json:"linkScenesSchedule"` } `json:"cron"` Storage struct { MatchOhash bool `default:"false" json:"match_ohash"` diff --git a/pkg/config/scraper_list.go b/pkg/config/scraper_list.go index 3ebb82586..29296522d 100644 --- a/pkg/config/scraper_list.go +++ b/pkg/config/scraper_list.go @@ -33,12 +33,13 @@ type CustomScrapers struct { VrphubScrapers []ScraperConfig `json:"vrphub"` } type ScraperConfig struct { - ID string `json:"-"` - URL string `json:"url"` - Name string `json:"name"` - Company string `json:"company"` - AvatarUrl string `json:"avatar_url"` - FileID string `json:"id,omitempty"` + ID string `json:"-"` + URL string `json:"url"` + Name string `json:"name"` + Company string `json:"company"` + AvatarUrl string `json:"avatar_url"` + FileID string `json:"id,omitempty"` + MasterSiteId string `json:"master_site_id,omitempty"` } var loadLock sync.Mutex @@ -117,7 +118,7 @@ func CheckMatchingSite(findSite ScraperConfig, searchList []ScraperConfig) bool if !strings.HasSuffix(s2, "/") { s2 += "/" } - if s1 == s2 { + if s1 == s2 && customSite.MasterSiteId == findSite.MasterSiteId { return true } } diff --git a/pkg/config/scrapers.json b/pkg/config/scrapers.json index f2a7dc0c2..d4bee433c 100644 --- a/pkg/config/scrapers.json +++ b/pkg/config/scrapers.json @@ -179,7 +179,7 @@ "url": "https://www.sexlikereal.com/studios/fuckpassvr", "name": "FuckPassVR", "company": "FuckPassVR", - "avatar_url": "https://cdn-vr.sexlikereal.com/images/studio_creatives/logotypes/1/352/logo_crop_1635153994.png" + "avatar_url": "https://cdn-vr.sexlikereal.com/images/studio_creatives/logotypes/1/352/logo_crop_1635153994.png" }, { "url": "https://www.sexlikereal.com/studios/heathering", @@ -197,7 +197,7 @@ "url": "https://www.sexlikereal.com/studios/jackandjillvr", "name": "JackandJillVR", "company": "JackandJillVR", - "avatar_url": "https://cdn-vr.sexlikereal.com/images/studio_creatives/logotypes/1/367/logo_crop_1645997567.png" + "avatar_url": "https://cdn-vr.sexlikereal.com/images/studio_creatives/logotypes/1/367/logo_crop_1645997567.png" }, { "url": "https://www.sexlikereal.com/studios/jimmydraws", diff --git a/pkg/externalreference/stashdb.go b/pkg/externalreference/stashdb.go index 22b035bb8..31d8572a9 100644 --- a/pkg/externalreference/stashdb.go +++ b/pkg/externalreference/stashdb.go @@ -532,7 +532,6 @@ func ReverseMatch() { var data models.StashPerformer json.Unmarshal([]byte(extref.ExternalData), &data) UpdateXbvrActor(data, actor.ID) - log.Info("match") } break sceneLoop } diff --git a/pkg/migrations/migrations.go b/pkg/migrations/migrations.go index 06ba28da9..035fbed79 100644 --- a/pkg/migrations/migrations.go +++ b/pkg/migrations/migrations.go @@ -776,6 +776,32 @@ func Migrate() { return tx.AutoMigrate(Site{}).Error }, }, + { + ID: "0076-Scene-Alt-Sources", + Migrate: func(tx *gorm.DB) error { + type Site struct { + MasterSiteID string `json:"master_site_id" xbvrbackup:"master_site_id"` + MatchingParams string `json:"matching_params" gorm:"size:1000" xbvrbackup:"matching_params"` + } + type ExternalReference struct { + UdfBool1 bool `json:"udf_bool1" xbvrbackup:"udf_bool1"` // user defined fields, use depends what type of data the extref is for. + UdfBool2 bool `json:"udf_bool2" xbvrbackup:"udf_bool2"` + UdfDatetime1 time.Time `json:"udf_datetime1" xbvrbackup:"udf_datetime1"` + } + type ExternalReferenceLink struct { + UdfDatetime1 time.Time `json:"udf_datetime1" xbvrbackup:"udf_datetime1"` + } + err := tx.AutoMigrate(Site{}).Error + if err != nil { + return err + } + err = tx.AutoMigrate(ExternalReferenceLink{}).Error + if err != nil { + return err + } + return tx.AutoMigrate(ExternalReference{}).Error + }, + }, // =============================================================================================== // Put DB Schema migrations above this line and migrations that rely on the updated schema below @@ -1503,7 +1529,7 @@ func Migrate() { } // backup bundle common.Log.Infof("Creating pre-migration backup, please waiit, backups can take some time on a system with a large number of scenes ") - tasks.BackupBundle(true, false, true, true, true, true, true, true, true, true, true, true, true, true, true, false, "0", "xbvr-premigration-bundle.json", "2") + tasks.BackupBundle(true, false, true, true, true, true, true, true, true, true, true, true, true, true, true, false, "", "0", "xbvr-premigration-bundle.json", "2") common.Log.Infof("Go to download/xbvr-premigration-bundle.json, or http://xxx.xxx.xxx.xxx:9999/download/xbvr-premigration-bundle.json if you need access to the backup") var sites []models.Site officalSiteChanges := []SiteChange{ diff --git a/pkg/models/model_external_reference.go b/pkg/models/model_external_reference.go index 8598e096a..133876af2 100644 --- a/pkg/models/model_external_reference.go +++ b/pkg/models/model_external_reference.go @@ -27,7 +27,11 @@ type ExternalReference struct { ExternalURL string `json:"external_url" gorm:"size:1000" xbvrbackup:"external_url"` ExternalDate time.Time `json:"external_date" xbvrbackup:"external_date"` ExternalData string `json:"external_data" sql:"type:text;" xbvrbackup:"external_data"` + UdfBool1 bool `json:"udf_bool1" xbvrbackup:"udf_bool1"` // user defined fields, use depends what type of data the extref is for. + UdfBool2 bool `json:"udf_bool2" xbvrbackup:"udf_bool2"` + UdfDatetime1 time.Time `json:"udf_datetime1" xbvrbackup:"udf_datetime1"` } + type ExternalReferenceLink struct { ID uint `gorm:"primary_key" json:"id" xbvrbackup:"-"` CreatedAt time.Time `json:"-" xbvrbackup:"created_at-"` @@ -36,10 +40,11 @@ type ExternalReferenceLink struct { InternalDbId uint `json:"internal_db_id" gorm:"index" xbvrbackup:"-"` InternalNameId string `json:"internal_name_id" gorm:"index" xbvrbackup:"internal_name_id"` - ExternalReferenceID uint `json:"external_reference_id" gorm:"index" xbvrbackup:"-"` - ExternalSource string `json:"external_source" xbvrbackup:"-"` - ExternalId string `json:"external_id" gorm:"index" xbvrbackup:"-"` - MatchType int `json:"match_type" xbvrbackup:"match_type"` + ExternalReferenceID uint `json:"external_reference_id" gorm:"index" xbvrbackup:"-"` + ExternalSource string `json:"external_source" xbvrbackup:"-"` + ExternalId string `json:"external_id" gorm:"index" xbvrbackup:"-"` + MatchType int `json:"match_type" xbvrbackup:"match_type"` + UdfDatetime1 time.Time `json:"udf_datetime1" xbvrbackup:"udf_datetime1"` ExternalReference ExternalReference `json:"external_reference" gorm:"foreignKey:ExternalReferenceId" xbvrbackup:"-"` } @@ -107,6 +112,23 @@ func (o *ExternalReference) FindExternalId(externalSource string, externalId str return commonDb.Preload("XbvrLinks").Where(&ExternalReference{ExternalSource: externalSource, ExternalId: externalId}).First(o).Error } +func (o *ExternalReferenceLink) FindByInternalID(internalTable string, internalId uint) []ExternalReferenceLink { + commonDb, _ := GetCommonDB() + var refs []ExternalReferenceLink + commonDb.Preload("ExternalReference").Where(&ExternalReferenceLink{InternalTable: internalTable, InternalDbId: internalId}).Find(&refs) + return refs +} +func (o *ExternalReferenceLink) FindByInternalName(internalTable string, internalName string) []ExternalReferenceLink { + commonDb, _ := GetCommonDB() + var refs []ExternalReferenceLink + commonDb.Preload("ExternalReference").Where(&ExternalReferenceLink{InternalTable: internalTable, InternalNameId: internalName}).Find(&refs) + return refs +} +func (o *ExternalReferenceLink) FindByExternaID(externalSource string, externalId string) { + commonDb, _ := GetCommonDB() + commonDb.Preload("ExternalReference").Where(&ExternalReferenceLink{ExternalSource: externalSource, ExternalId: externalId}).Find(&o) +} + func (o *ExternalReference) Save() { commonDb, _ := GetCommonDB() @@ -129,6 +151,11 @@ func (o *ExternalReference) Delete() { commonDb.Delete(&o) } +func (o *ExternalReferenceLink) Delete() { + commonDb, _ := GetCommonDB() + commonDb.Delete(&o) +} + func (o *ExternalReference) AddUpdateWithUrl() { commonDb, _ := GetCommonDB() diff --git a/pkg/models/model_scene.go b/pkg/models/model_scene.go index 9ffcfd220..8a3ae6305 100644 --- a/pkg/models/model_scene.go +++ b/pkg/models/model_scene.go @@ -114,6 +114,8 @@ type Scene struct { Description string `gorm:"-" json:"description" xbvrbackup:"-"` Score float64 `gorm:"-" json:"_score" xbvrbackup:"-"` + + AlternateSource []ExternalReferenceLink `json:"alternate_source" xbvrbackup:"-"` } type Image struct { @@ -131,6 +133,12 @@ type VideoSource struct { Quality string `json:"quality"` } +type Config struct { + Advanced struct { + UseAltSrcInFileMatching bool `json:"useAltSrcInFileMatching"` + } `json:"advanced"` +} + func (i *Scene) Save() error { commonDb, _ := GetCommonDB() @@ -404,14 +412,7 @@ func SceneCreateUpdateFromExternal(db *gorm.DB, ext ScrapedScene) error { var o Scene db.Where(&Scene{SceneID: ext.SceneID}).FirstOrCreate(&o) - o.NeedsUpdate = false - o.EditsApplied = false - o.SceneID = ext.SceneID - o.ScraperId = ext.ScraperID - if o.Title != ext.Title { - o.Title = ext.Title - // reset scriptfile.IsExported state on title change scriptfiles, err := o.GetScriptFiles() if err == nil { @@ -424,6 +425,71 @@ func SceneCreateUpdateFromExternal(db *gorm.DB, ext ScrapedScene) error { } } + o.PopulateSceneFieldsFromExternal(db, ext) + var site Site + db.Where("id = ?", o.ScraperId).FirstOrInit(&site) + o.IsSubscribed = site.Subscribed + SaveWithRetry(db, &o) + + // Clean & Associate Tags + var tags = o.Tags + db.Model(&o).Association("Tags").Clear() + for _, tag := range tags { + tmpTag := Tag{} + db.Where(&Tag{Name: tag.Name}).FirstOrCreate(&tmpTag) + db.Model(&o).Association("Tags").Append(tmpTag) + } + + // Clean & Associate Actors + db.Model(&o).Association("Cast").Clear() + var tmpActor Actor + for _, name := range ext.Cast { + tmpActor = Actor{} + db.Where(&Actor{Name: strings.Replace(name, ".", "", -1)}).FirstOrCreate(&tmpActor) + saveActor := false + if ext.ActorDetails[name].ImageUrl != "" { + if tmpActor.ImageUrl == "" { + tmpActor.ImageUrl = ext.ActorDetails[name].ImageUrl + saveActor = true + } + if tmpActor.AddToImageArray(ext.ActorDetails[name].ImageUrl) { + saveActor = true + } + } + if ext.ActorDetails[name].ProfileUrl != "" { + if tmpActor.AddToActorUrlArray(ActorLink{Url: ext.ActorDetails[name].ProfileUrl, Type: ext.ActorDetails[name].Source}) { + saveActor = true + } + } + if saveActor { + tmpActor.Save() + } + db.Model(&o).Association("Cast").Append(tmpActor) + } + // delete any altrernate scene records, in case this scene was originally a linked scene + var extrefs []ExternalReference + db.Where("external_source like 'alternate scene %' and external_url = ?", o.SceneURL).Find(&extrefs) + for _, extref := range extrefs { + db.Where("external_reference_id = ?", extref.ID).Delete(&ExternalReferenceLink{}) + db.Delete(&extref) + } + + return nil +} + +func (o *Scene) PopulateSceneFieldsFromExternal(db *gorm.DB, ext ScrapedScene) { + // this function is shared between scenes and alternate scenes, + // it should only setup values in the scene record from the scraped scene + // it should not update scene data, as that won't apply for alternate scene sources + if ext.SceneID == "" { + return + } + + o.NeedsUpdate = false + o.EditsApplied = false + o.SceneID = ext.SceneID + o.ScraperId = ext.ScraperID + o.Title = ext.Title o.SceneType = ext.SceneType o.Studio = ext.Studio o.Site = ext.Site @@ -489,62 +555,58 @@ func SceneCreateUpdateFromExternal(db *gorm.DB, ext ScrapedScene) error { var site Site db.Where("id = ?", o.ScraperId).FirstOrInit(&site) o.IsSubscribed = site.Subscribed - SaveWithRetry(db, &o) - // Clean & Associate Tags - db.Model(&o).Association("Tags").Clear() - var tmpTag Tag + var tags []Tag for _, name := range ext.Tags { tagClean := ConvertTag(name) if tagClean != "" { - tmpTag = Tag{} - db.Where(&Tag{Name: tagClean}).FirstOrCreate(&tmpTag) - db.Model(&o).Association("Tags").Append(tmpTag) + tags = append(tags, Tag{Name: tagClean}) } } + o.Tags = tags // Clean & Associate Actors - db.Model(&o).Association("Cast").Clear() + var cast []Actor var tmpActor Actor for _, name := range ext.Cast { tmpActor = Actor{} db.Where(&Actor{Name: strings.Replace(name, ".", "", -1)}).FirstOrCreate(&tmpActor) - saveActor := false - if ext.ActorDetails[name].ImageUrl != "" { - if tmpActor.ImageUrl == "" { - tmpActor.ImageUrl = ext.ActorDetails[name].ImageUrl - saveActor = true - } - if tmpActor.AddToImageArray(ext.ActorDetails[name].ImageUrl) { - saveActor = true - } - // AddActionActor(name, ext.ActorDetails[name].Source, "add", "image_url", ext.ActorDetails[name].ImageUrl) - } - if ext.ActorDetails[name].ProfileUrl != "" { - if tmpActor.AddToActorUrlArray(ActorLink{Url: ext.ActorDetails[name].ProfileUrl, Type: ext.ActorDetails[name].Source}) { - saveActor = true - } - // AddActionActor(name, ext.ActorDetails[name].Source, "add", "image_url", ext.ActorDetails[name].ImageUrl) - } - if saveActor { - tmpActor.Save() - } - db.Model(&o).Association("Cast").Append(tmpActor) + cast = append(cast, tmpActor) } - - return nil + o.Cast = cast } func SceneUpdateScriptData(db *gorm.DB, ext ScrapedScene) { - var o Scene - o.GetIfExistByPK(ext.InternalSceneId) - - if o.ID != 0 { - if o.ScriptPublished.IsZero() || o.HumanScript != ext.HumanScript || o.AiScript != ext.AiScript { - o.ScriptPublished = time.Now() - o.HumanScript = ext.HumanScript - o.AiScript = ext.AiScript - o.Save() + if ext.MasterSiteId == "" { + var o Scene + o.GetIfExistByPK(ext.InternalSceneId) + + if o.ID != 0 { + if o.ScriptPublished.IsZero() || o.HumanScript != ext.HumanScript || o.AiScript != ext.AiScript { + o.ScriptPublished = time.Now() + o.HumanScript = ext.HumanScript + o.AiScript = ext.AiScript + o.Save() + } + } + } else { + var extref ExternalReference + extref.FindExternalId("alternate scene "+ext.ScraperID, ext.SceneID) + var externalData SceneAlternateSource + json.Unmarshal([]byte(extref.ExternalData), &externalData) + if extref.ID > 0 { + if externalData.Scene.ScriptPublished.IsZero() || externalData.Scene.HumanScript != ext.HumanScript || externalData.Scene.AiScript != ext.AiScript { + // set user defined fields for querying, rather than querying the json + extref.UdfDatetime1 = time.Now() + extref.UdfBool1 = ext.HumanScript + extref.UdfBool2 = ext.AiScript + externalData.Scene.ScriptPublished = extref.UdfDatetime1 + externalData.Scene.HumanScript = ext.HumanScript + externalData.Scene.AiScript = ext.AiScript + newjson, _ := json.Marshal(externalData) + extref.ExternalData = string(newjson) + extref.Save() + } } } } @@ -661,6 +723,9 @@ func QuerySceneSummaries(r RequestSceneList) []SceneSummary { } func queryScenes(db *gorm.DB, r RequestSceneList) (*gorm.DB, *gorm.DB) { + // get config, can't reference config directly due to circular package references + config := getConfig(db) + tx := db.Model(&Scene{}) if r.IsWatched.Present() { @@ -818,13 +883,36 @@ func queryScenes(db *gorm.DB, r RequestSceneList) (*gorm.DB, *gorm.DB) { case "VRPorn Scraper": where = `scenes.scene_id like "vrporn-%"` case "Has Script Download": - where = "scenes.script_published > '0001-01-01 00:00:00+00:00'" + // querying the scenes in from alternate sources (stored in external_reference) has a performance impact, so it's user choice + if config.Advanced.UseAltSrcInFileMatching { + where = "(scenes.script_published > '0001-01-01 00:00:00+00:00' or (select distinct 1 from external_reference_links erl join external_references er on er.id=erl.external_reference_id where erl.internal_table='scenes' and internal_db_id=scenes.id and er.udf_datetime1 > '0001-01-02'))" + } else { + where = "scenes.script_published > '0001-01-01 00:00:00+00:00'" + } case "Has AI Generated Script": - where = "scenes.ai_script = 1" + // querying the scenes in from alternate sources (stored in external_reference) has a performance impact, so it's user choice + if config.Advanced.UseAltSrcInFileMatching { + where = "(scenes.ai_script = 1 or (select distinct 1 from external_reference_links erl join external_references er on er.id=erl.external_reference_id where erl.internal_table='scenes' and internal_db_id=scenes.id and JSON_EXTRACT(er.external_data, '$.scene.ai_script') = 1))" + } else { + where = "scenes.ai_script = 1" + } case "Has Human Generated Script": - where = "scenes.human_script = 1" + // querying the scenes in from alternate sources (stored in external_reference) has a performance impact, so it's user choice + if config.Advanced.UseAltSrcInFileMatching { + where = "(scenes.human_script = 1 or (select distinct 1 from external_reference_links erl join external_references er on er.id=erl.external_reference_id where erl.internal_table='scenes' and internal_db_id=scenes.id and JSON_EXTRACT(er.external_data, '$.scene.human_script') = 1))" + } else { + where = "scenes.human_script = 1" + } case "Has Favourite Actor": where = "exists (select * from scene_cast join actors on actors.id=scene_cast.actor_id where actors.favourite=1 and scene_cast.scene_id=scenes.id)" + case "Available from POVR": + where = "exists (select 1 from external_reference_links where external_source like 'alternate scene %' and external_id like 'povr-%' and internal_db_id = scenes.id)" + case "Available from VRPorn": + where = "exists (select 1 from external_reference_links where external_source like 'alternate scene %' and external_id like 'vrporn-%' and internal_db_id = scenes.id)" + case "Available from SLR": + where = "exists (select 1 from external_reference_links where external_source like 'alternate scene %' and external_id like 'slr-%' and internal_db_id = scenes.id)" + case "Available from Alternate Sites": + where = "exists (select 1 from external_reference_links where external_source like 'alternate scene %' and internal_db_id = scenes.id)" } if negate { @@ -1037,7 +1125,16 @@ func queryScenes(db *gorm.DB, r RequestSceneList) (*gorm.DB, *gorm.DB) { case "scene_updated_desc": tx = tx.Order("updated_at desc") case "script_published_desc": - tx = tx.Order("script_published desc") + // querying the scenes in from alternate sources (stored in external_reference) has a performance impact, so it's user choice + if config.Advanced.UseAltSrcInFileMatching { + tx = tx.Order(` + case when script_published > (select max(er.udf_datetime1) from external_reference_links erl join external_references er on er.id=erl.external_reference_id where erl.internal_table='scenes' and erl.internal_db_id=scenes.id and er.external_source like 'alternate scene %') + then script_published + else (select max(er.udf_datetime1) from external_reference_links erl join external_references er on er.id=erl.external_reference_id where erl.internal_table='scenes' and erl.internal_db_id=scenes.id and er.external_source like 'alternate scene %') + end desc`) + } else { + tx = tx.Order("script_published desc") + } case "scene_id_desc": tx = tx.Order("scene_id desc") case "random": @@ -1046,6 +1143,9 @@ func queryScenes(db *gorm.DB, r RequestSceneList) (*gorm.DB, *gorm.DB) { } else { tx = tx.Order("random()") } + case "alt_src_desc": + //tx = tx.Order(`(select max(er.external_date) from external_reference_links erl join external_references er on er.id=erl.external_reference_id where erl.internal_table='scenes' and erl.internal_db_id=scenes.id and er.external_source like 'alternate scene %') desc`) + tx = tx.Order(`(select max(erl.udf_datetime1) from external_reference_links erl where erl.internal_table='scenes' and erl.internal_db_id=scenes.id and erl.external_source like 'alternate scene %') desc`) default: tx = tx.Order("release_date desc") } @@ -1090,3 +1190,13 @@ func setCuepointString(cuepoint string) string { return "%" + cuepoint + "%" } } + +func getConfig(db *gorm.DB) Config { + var config Config + + var kv KV + db.Where("`key`='config'").First(&kv) + + json.Unmarshal([]byte(kv.Value), &config) + return config +} diff --git a/pkg/models/model_scene_alternate_source.go b/pkg/models/model_scene_alternate_source.go new file mode 100644 index 000000000..78cdfa0c3 --- /dev/null +++ b/pkg/models/model_scene_alternate_source.go @@ -0,0 +1,109 @@ +package models + +import ( + "encoding/json" + "strings" + "time" +) + +// SceneCuepoint data model +type SceneAlternateSource struct { + MasterSiteId string `json:"master_site_id"` + MatchAchieved int `json:"MatchAchieved"` + Query string `json:"query"` // ******* remove before go live, just to help testing query used for the scene + Scene Scene `json:"scene"` +} +type AltSrcMatchParams struct { + IgnoreReleasedBefore time.Time `json:"ignore_released_before"` // optionally don't check the released date, if before a certain date, this may be useful to only search on release dates after an inital load of scene on a site + DelayLinking int `json:"delay_linking"` // allows delaying linking from the release date. Useful for LethalHardcore which is often released on their site after SLR, etc, which would result in linking to another scene + ReprocessLinks int `json:"reprocess_links"` // will reprocess linking for the specified number of days. Another approach for LethalHardcore. + ReleasedMatchType string `json:"released_match_type"` // must, should, do not + ReleasedPrior int `json:"released_prior"` // match on where the scene release date within x days prior to the release date of the alternate scene, eg a studio may only release on other sites after 90 days + ReleasedAfter int `json:"released_after"` // do not match scenes befoew a certain date + DurationMatchType string `json:"duration_match_type"` // must, should, do not + DurationMin int `json:"duration_min"` // ignore if the scene duration is below a specific value, may be useful for trailers + DurationRangeLess int `json:"duration_range_less"` // match on where the scene duration is more than x seconds less than the duration of the alternate scene + DurationRangeMore int `json:"duration_range_more"` // match on where the scene duration is less than x seconds more than the duration of the alternate scene + CastMatchType string `json:"cast_match_type"` // should or do not. Note "must" match is not an option for cast members due to alises and sites using different names. + DescriptionMatchType string `json:"desc_match_type"` // should, do not. "Must" is not an option + BoostTitleExact float32 `json:"boost_title"` // boosting allows you control the revalant significant between search fields, default is 1. Boostig has no effect on "must" matches + BoostTitleAnyWords float32 `json:"boost_title_any_words"` // boosting allows you control the revalant significant between search fields, default is 1. Boostig has no effect on "must" matches + BoostReleased float32 `json:"boost_released"` + BoostCast float32 `json:"boost_cast"` + BoostDescription float32 `json:"boost_description"` +} + +const defaultMatchType = "should" +const releasedPrior = 14 +const releasedAfter = 7 +const durationMin = 3 +const boostTitleExact = 4.0 +const boostTitleAnyWords = 1.5 +const boostReleased = 2 +const boostCast = 2 +const boostDescription = 0.25 + +func (p *AltSrcMatchParams) Default() { + p.ReleasedMatchType = defaultMatchType + p.ReleasedPrior = releasedPrior + p.ReleasedAfter = releasedAfter + p.DurationMatchType = defaultMatchType + p.DurationMin = durationMin + p.CastMatchType = defaultMatchType + p.DescriptionMatchType = defaultMatchType + p.BoostReleased = boostReleased + p.BoostTitleExact = boostTitleExact + p.BoostTitleAnyWords = boostTitleAnyWords + p.BoostCast = boostCast + p.BoostDescription = boostDescription + p.BoostReleased = boostReleased +} + +func (p *AltSrcMatchParams) UnmarshalParams(jsonStr string) error { + if jsonStr == "" { + p.Default() + return nil + } else { + err := json.Unmarshal([]byte(jsonStr), &p) + if err != nil { + return err + } + if !strings.Contains(jsonStr, "released_match_type") { + p.ReleasedMatchType = defaultMatchType + } + if !strings.Contains(jsonStr, "released_prior") { + p.ReleasedPrior = releasedPrior + } + if !strings.Contains(jsonStr, "released_after") { + p.ReleasedAfter = releasedAfter + } + if !strings.Contains(jsonStr, "duration_match_type") { + p.DurationMatchType = defaultMatchType + } + if !strings.Contains(jsonStr, "duration_min") { + p.DurationMin = durationMin + } + if !strings.Contains(jsonStr, "cast_match_type") { + p.CastMatchType = defaultMatchType + } + if !strings.Contains(jsonStr, "desc_match_type") { + p.DescriptionMatchType = defaultMatchType + } + if !strings.Contains(jsonStr, "boost_title") { + p.BoostTitleExact = boostTitleExact + } + if !strings.Contains(jsonStr, "boost_title_any_words") { + p.BoostTitleAnyWords = boostTitleAnyWords + } + if !strings.Contains(jsonStr, "boost_released") { + p.BoostCast = boostReleased + } + if !strings.Contains(jsonStr, "boost_cast") { + p.BoostCast = boostCast + } + if !strings.Contains(jsonStr, "boost_description") { + p.BoostDescription = boostDescription + } + } + return nil +} diff --git a/pkg/models/model_scraper.go b/pkg/models/model_scraper.go index 0626e348f..6d5bf44ec 100644 --- a/pkg/models/model_scraper.go +++ b/pkg/models/model_scraper.go @@ -10,11 +10,12 @@ var scrapers []Scraper type ScraperFunc func(*sync.WaitGroup, bool, []string, chan<- ScrapedScene, string, string, bool) error type Scraper struct { - ID string `json:"id"` - Name string `json:"name"` - AvatarURL string `json:"avaatarurl"` - Domain string `json:"domain"` - Scrape ScraperFunc `json:"-"` + ID string `json:"id"` + Name string `json:"name"` + AvatarURL string `json:"avaatarurl"` + Domain string `json:"domain"` + Scrape ScraperFunc `json:"-"` + MasterSiteId string `json:"master_site_id"` } type ScrapedScene struct { @@ -46,6 +47,7 @@ type ScrapedScene struct { InternalSceneId uint `json:"internal_id"` ActorDetails map[string]ActorDetails `json:"actor_details"` + MasterSiteId string `json:"master_site_id"` } type ActorDetails struct { @@ -78,12 +80,13 @@ func GetScrapers() []Scraper { return scrapers } -func RegisterScraper(id string, name string, avatarURL string, domain string, f ScraperFunc) { +func RegisterScraper(id string, name string, avatarURL string, domain string, f ScraperFunc, masterSiteId string) { s := Scraper{} s.ID = id s.Name = name s.AvatarURL = avatarURL s.Domain = domain s.Scrape = f + s.MasterSiteId = masterSiteId scrapers = append(scrapers, s) } diff --git a/pkg/models/model_site.go b/pkg/models/model_site.go index 9ed817c05..8af24250d 100644 --- a/pkg/models/model_site.go +++ b/pkg/models/model_site.go @@ -8,15 +8,17 @@ import ( ) type Site struct { - ID string `gorm:"primary_key" json:"id" xbvrbackup:"-"` - Name string `json:"name" xbvrbackup:"name"` - AvatarURL string `json:"avatar_url" xbvrbackup:"-"` - IsBuiltin bool `json:"is_builtin" xbvrbackup:"-"` - IsEnabled bool `json:"is_enabled" xbvrbackup:"is_enabled"` - LastUpdate time.Time `json:"last_update" xbvrbackup:"-"` - Subscribed bool `json:"subscribed" xbvrbackup:"subscribed"` - HasScraper bool `gorm:"-" json:"has_scraper" xbvrbackup:"-"` - LimitScraping bool `json:"limit_scraping" xbvrbackup:"limit_scraping"` + ID string `gorm:"primary_key" json:"id" xbvrbackup:"-"` + Name string `json:"name" xbvrbackup:"name"` + AvatarURL string `json:"avatar_url" xbvrbackup:"-"` + IsBuiltin bool `json:"is_builtin" xbvrbackup:"-"` + IsEnabled bool `json:"is_enabled" xbvrbackup:"is_enabled"` + LastUpdate time.Time `json:"last_update" xbvrbackup:"-"` + Subscribed bool `json:"subscribed" xbvrbackup:"subscribed"` + HasScraper bool `gorm:"-" json:"has_scraper" xbvrbackup:"-"` + LimitScraping bool `json:"limit_scraping" xbvrbackup:"limit_scraping"` + MasterSiteID string `json:"master_site_id" xbvrbackup:"master_site_id"` + MatchingParams string `json:"matching_params" gorm:"size:1000" xbvrbackup:"matching_params"` } func (i *Site) Save() error { @@ -59,6 +61,7 @@ func InitSites() { st.Name = scrapers[i].Name st.AvatarURL = scrapers[i].AvatarURL st.IsBuiltin = true + st.MasterSiteID = scrapers[i].MasterSiteId st.Save() } } diff --git a/pkg/scrape/badoink.go b/pkg/scrape/badoink.go index f28021581..42d0b19fa 100644 --- a/pkg/scrape/badoink.go +++ b/pkg/scrape/badoink.go @@ -31,6 +31,8 @@ func BadoinkSite(wg *sync.WaitGroup, updateSite bool, knownScenes []string, out siteCollector := createCollector("badoinkvr.com", "babevr.com", "vrcosplayx.com", "18vr.com", "kinkvr.com") trailerCollector := cloneCollector(sceneCollector) + commonDb, _ := models.GetCommonDB() + sceneCollector.OnHTML(`html`, func(e *colly.HTMLElement) { sc := models.ScrapedScene{} sc.ScraperID = scraperID @@ -211,7 +213,7 @@ func BadoinkSite(wg *sync.WaitGroup, updateSite bool, knownScenes []string, out e.ForEach(`a[href^="/category/funscript"]`, func(id int, e *colly.HTMLElement) { var existingScene models.Scene - existingScene.GetIfExistURL(sceneURL) + commonDb.Where(&models.Scene{SceneURL: sceneURL}).First(&existingScene) if existingScene.ID != 0 && existingScene.ScriptPublished.IsZero() { var sc models.ScrapedScene sc.InternalSceneId = existingScene.ID diff --git a/pkg/scrape/czechvr.go b/pkg/scrape/czechvr.go index 1baa6a055..dc9b4ad4b 100644 --- a/pkg/scrape/czechvr.go +++ b/pkg/scrape/czechvr.go @@ -17,6 +17,7 @@ import ( func CzechVR(wg *sync.WaitGroup, updateSite bool, knownScenes []string, out chan<- models.ScrapedScene, singleSceneURL string, scraperID string, siteID string, nwID string, singeScrapeAdditionalInfo string, limitScraping bool) error { defer wg.Done() logScrapeStart(scraperID, siteID) + commonDb, _ := models.GetCommonDB() sceneCollector := createCollector("www.czechvrnetwork.com") siteCollector := createCollector("www.czechvrnetwork.com") @@ -167,7 +168,7 @@ func CzechVR(wg *sync.WaitGroup, updateSite bool, knownScenes []string, out chan if config.Config.Funscripts.ScrapeFunscripts { e.ForEach(`div.iconinteractive`, func(id int, e *colly.HTMLElement) { var existingScene models.Scene - existingScene.GetIfExistURL(sceneURL) + commonDb.Where(&models.Scene{SceneURL: sceneURL}).First(&existingScene) if existingScene.ID != 0 && existingScene.ScriptPublished.IsZero() { var sc models.ScrapedScene sc.InternalSceneId = existingScene.ID diff --git a/pkg/scrape/povr.go b/pkg/scrape/povr.go index dc6e60b6c..a02870b58 100644 --- a/pkg/scrape/povr.go +++ b/pkg/scrape/povr.go @@ -15,7 +15,7 @@ import ( "github.com/xbapps/xbvr/pkg/models" ) -func POVR(wg *sync.WaitGroup, updateSite bool, knownScenes []string, out chan<- models.ScrapedScene, singleSceneURL string, scraperID string, siteID string, company string, siteURL string, singeScrapeAdditionalInfo string, limitScraping bool) error { +func POVR(wg *sync.WaitGroup, updateSite bool, knownScenes []string, out chan<- models.ScrapedScene, singleSceneURL string, scraperID string, siteID string, company string, siteURL string, singeScrapeAdditionalInfo string, limitScraping bool, masterSiteId string) error { defer wg.Done() logScrapeStart(scraperID, siteID) @@ -29,6 +29,7 @@ func POVR(wg *sync.WaitGroup, updateSite bool, knownScenes []string, out chan<- sc.Studio = company sc.Site = siteID sc.HomepageURL = strings.Split(e.Request.URL.String(), "?")[0] + sc.MasterSiteId = masterSiteId if scraperID == "" { // there maybe no site/studio if user is jusy scraping a scene url @@ -123,7 +124,7 @@ func POVR(wg *sync.WaitGroup, updateSite bool, knownScenes []string, out chan<- siteCollector.OnHTML(`div.thumbnail-wrap div.thumbnail a.thumbnail__link`, func(e *colly.HTMLElement) { sceneURL := e.Request.AbsoluteURL(e.Attr("href")) - // If scene exist in database, there's no need to scrape + // If scene exists in database, or the slternate source exists, there's no need to scrape if !funk.ContainsString(knownScenes, sceneURL) && !strings.Contains(sceneURL, "/join") { sceneCollector.Visit(sceneURL) } @@ -149,7 +150,7 @@ func POVR(wg *sync.WaitGroup, updateSite bool, knownScenes []string, out chan<- return nil } -func addPOVRScraper(id string, name string, company string, avatarURL string, custom bool, siteURL string) { +func addPOVRScraper(id string, name string, company string, avatarURL string, custom bool, siteURL string, masterSiteId string) { suffixedName := name siteNameSuffix := name if custom { @@ -158,21 +159,27 @@ func addPOVRScraper(id string, name string, company string, avatarURL string, cu } else { suffixedName += " (POVR)" } - registerScraper(id, suffixedName, avatarURL, "povr.com", func(wg *sync.WaitGroup, updateSite bool, knownScenes []string, out chan<- models.ScrapedScene, singleSceneURL string, singeScrapeAdditionalInfo string, limitScraping bool) error { - return POVR(wg, updateSite, knownScenes, out, singleSceneURL, id, siteNameSuffix, company, siteURL, singeScrapeAdditionalInfo, limitScraping) - }) + if masterSiteId == "" { + registerScraper(id, suffixedName, avatarURL, "povr.com", func(wg *sync.WaitGroup, updateSite bool, knownScenes []string, out chan<- models.ScrapedScene, singleSceneURL string, singeScrapeAdditionalInfo string, limitScraping bool) error { + return POVR(wg, updateSite, knownScenes, out, singleSceneURL, id, siteNameSuffix, company, siteURL, singeScrapeAdditionalInfo, limitScraping, "") + }) + } else { + registerAlternateScraper(id, suffixedName, avatarURL, "povr.com", masterSiteId, func(wg *sync.WaitGroup, updateSite bool, knownScenes []string, out chan<- models.ScrapedScene, singleSceneURL string, singeScrapeAdditionalInfo string, limitScraping bool) error { + return POVR(wg, updateSite, knownScenes, out, singleSceneURL, id, siteNameSuffix, company, siteURL, singeScrapeAdditionalInfo, limitScraping, masterSiteId) + }) + } } func init() { registerScraper("povr-single_scene", "POVR - Other Studios", "", "povr.com", func(wg *sync.WaitGroup, updateSite bool, knownScenes []string, out chan<- models.ScrapedScene, singleSceneURL string, singeScrapeAdditionalInfo string, limitScraping bool) error { - return POVR(wg, updateSite, knownScenes, out, singleSceneURL, "", "", "", "", singeScrapeAdditionalInfo, limitScraping) + return POVR(wg, updateSite, knownScenes, out, singleSceneURL, "", "", "", "", singeScrapeAdditionalInfo, limitScraping, "") }) var scrapers config.ScraperList scrapers.Load() for _, scraper := range scrapers.XbvrScrapers.PovrScrapers { - addPOVRScraper(scraper.ID, scraper.Name, scraper.Company, scraper.AvatarUrl, false, scraper.URL) + addPOVRScraper(scraper.ID, scraper.Name, scraper.Company, scraper.AvatarUrl, false, scraper.URL, scraper.MasterSiteId) } for _, scraper := range scrapers.CustomScrapers.PovrScrapers { - addPOVRScraper(scraper.ID, scraper.Name, scraper.Company, scraper.AvatarUrl, true, scraper.URL) + addPOVRScraper(scraper.ID, scraper.Name, scraper.Company, scraper.AvatarUrl, true, scraper.URL, scraper.MasterSiteId) } } diff --git a/pkg/scrape/scrape.go b/pkg/scrape/scrape.go index 118556f6c..871b5feaa 100644 --- a/pkg/scrape/scrape.go +++ b/pkg/scrape/scrape.go @@ -84,7 +84,12 @@ func getScrapeCacheDir() string { } func registerScraper(id string, name string, avatarURL string, domain string, f models.ScraperFunc) { - models.RegisterScraper(id, name, avatarURL, domain, f) + models.RegisterScraper(id, name, avatarURL, domain, f, "") +} + +func registerAlternateScraper(id string, name string, avatarURL string, domain string, masterSiteId string, f models.ScraperFunc) { + // alternate scrapers are to scrape scenes available at other sites to match against a scenes from the studio's site, eg scrape VRHush scenes from SLR and match to scenes from VRHush + models.RegisterScraper(id, name, avatarURL, domain, f, masterSiteId) } func logScrapeStart(id string, name string) { diff --git a/pkg/scrape/slrstudios.go b/pkg/scrape/slrstudios.go index f522b159c..87fca7886 100644 --- a/pkg/scrape/slrstudios.go +++ b/pkg/scrape/slrstudios.go @@ -1,6 +1,7 @@ package scrape import ( + "encoding/json" "html" "regexp" "strconv" @@ -15,7 +16,7 @@ import ( "github.com/xbapps/xbvr/pkg/models" ) -func SexLikeReal(wg *sync.WaitGroup, updateSite bool, knownScenes []string, out chan<- models.ScrapedScene, singleSceneURL string, scraperID string, siteID string, company string, siteURL string, singeScrapeAdditionalInfo string, limitScraping bool) error { +func SexLikeReal(wg *sync.WaitGroup, updateSite bool, knownScenes []string, out chan<- models.ScrapedScene, singleSceneURL string, scraperID string, siteID string, company string, siteURL string, singeScrapeAdditionalInfo string, limitScraping bool, masterSiteId string) error { defer wg.Done() logScrapeStart(scraperID, siteID) @@ -36,6 +37,7 @@ func SexLikeReal(wg *sync.WaitGroup, updateSite bool, knownScenes []string, out sc.SceneType = "VR" sc.Studio = company sc.Site = siteID + sc.MasterSiteId = masterSiteId if scraperID == "" { // there maybe no site/studio if user is jusy scraping a scene url e.ForEach(`div[data-qa="page-scene-studio-name"]`, func(id int, e *colly.HTMLElement) { @@ -280,8 +282,19 @@ func SexLikeReal(wg *sync.WaitGroup, updateSite bool, knownScenes []string, out if config.Config.Funscripts.ScrapeFunscripts { var existingScene models.Scene - existingScene.GetIfExistURL(sceneURL) - if existingScene.ID != 0 { + + if masterSiteId == "" { + commonDb.Where(&models.Scene{SceneURL: sceneURL}).First(&existingScene) + } else { + // get the scene from the external_reference table + var extref models.ExternalReference + extref.FindExternalUrl("alternate scene "+scraperID, sceneURL) + var externalData models.SceneAlternateSource + json.Unmarshal([]byte(extref.ExternalData), &externalData) + existingScene = externalData.Scene + } + + if existingScene.ID != 0 || masterSiteId != "" { fleshlightBadge := false aiBadge := false multiBadge := false @@ -312,16 +325,18 @@ func SexLikeReal(wg *sync.WaitGroup, updateSite bool, knownScenes []string, out if existingScene.HumanScript != human || existingScene.AiScript != ai { var sc models.ScrapedScene sc.InternalSceneId = existingScene.ID + sc.SceneID = existingScene.SceneID + sc.ScraperID = scraperID sc.HasScriptDownload = true sc.OnlyUpdateScriptData = true sc.HumanScript = human sc.AiScript = ai + sc.MasterSiteId = masterSiteId out <- sc } } } - // If scene exist in database, there's no need to scrape if !funk.ContainsString(knownScenes, sceneURL) { durationText := e.ChildText("div.c-grid-ratio-bottom.u-z--two") m := durationRegExForSceneCard.FindStringSubmatch(durationText) @@ -447,7 +462,7 @@ func appendFilenames(sc *models.ScrapedScene, siteID string, filenameRegEx *rege } } -func addSLRScraper(id string, name string, company string, avatarURL string, custom bool, siteURL string) { +func addSLRScraper(id string, name string, company string, avatarURL string, custom bool, siteURL string, masterSiteId string) { suffixedName := name siteNameSuffix := name if custom { @@ -463,23 +478,29 @@ func addSLRScraper(id string, name string, company string, avatarURL string, cus avatarURL = "https://www.sexlikereal.com/s/refactor/images/favicons/android-icon-192x192.png" } - registerScraper(id, suffixedName, avatarURL, "sexlikereal.com", func(wg *sync.WaitGroup, updateSite bool, knownScenes []string, out chan<- models.ScrapedScene, singleSceneURL string, singeScrapeAdditionalInfo string, limitScraping bool) error { - return SexLikeReal(wg, updateSite, knownScenes, out, singleSceneURL, id, siteNameSuffix, company, siteURL, singeScrapeAdditionalInfo, limitScraping) - }) + if masterSiteId == "" { + registerScraper(id, suffixedName, avatarURL, "sexlikereal.com", func(wg *sync.WaitGroup, updateSite bool, knownScenes []string, out chan<- models.ScrapedScene, singleSceneURL string, singeScrapeAdditionalInfo string, limitScraping bool) error { + return SexLikeReal(wg, updateSite, knownScenes, out, singleSceneURL, id, siteNameSuffix, company, siteURL, singeScrapeAdditionalInfo, limitScraping, "") + }) + } else { + registerAlternateScraper(id, suffixedName, avatarURL, "sexlikereal.com", masterSiteId, func(wg *sync.WaitGroup, updateSite bool, knownScenes []string, out chan<- models.ScrapedScene, singleSceneURL string, singeScrapeAdditionalInfo string, limitScraping bool) error { + return SexLikeReal(wg, updateSite, knownScenes, out, singleSceneURL, id, siteNameSuffix, company, siteURL, singeScrapeAdditionalInfo, limitScraping, masterSiteId) + }) + } } func init() { var scrapers config.ScraperList // scraper for single scenes with no existing scraper for the studio registerScraper("slr-single_scene", "SLR - Other Studios", "", "sexlikereal.com", func(wg *sync.WaitGroup, updateSite bool, knownScenes []string, out chan<- models.ScrapedScene, singleSceneURL string, singeScrapeAdditionalInfo string, limitScraping bool) error { - return SexLikeReal(wg, updateSite, knownScenes, out, singleSceneURL, "", "", "", "", singeScrapeAdditionalInfo, limitScraping) + return SexLikeReal(wg, updateSite, knownScenes, out, singleSceneURL, "", "", "", "", singeScrapeAdditionalInfo, limitScraping, "") }) scrapers.Load() for _, scraper := range scrapers.XbvrScrapers.SlrScrapers { - addSLRScraper(scraper.ID, scraper.Name, scraper.Company, scraper.AvatarUrl, false, scraper.URL) + addSLRScraper(scraper.ID, scraper.Name, scraper.Company, scraper.AvatarUrl, false, scraper.URL, scraper.MasterSiteId) } for _, scraper := range scrapers.CustomScrapers.SlrScrapers { - addSLRScraper(scraper.ID, scraper.Name, scraper.Company, scraper.AvatarUrl, true, scraper.URL) + addSLRScraper(scraper.ID, scraper.Name, scraper.Company, scraper.AvatarUrl, true, scraper.URL, scraper.MasterSiteId) } } diff --git a/pkg/scrape/vrporn.go b/pkg/scrape/vrporn.go index 97255fe74..3cde9c9c1 100644 --- a/pkg/scrape/vrporn.go +++ b/pkg/scrape/vrporn.go @@ -14,7 +14,7 @@ import ( "github.com/xbapps/xbvr/pkg/models" ) -func VRPorn(wg *sync.WaitGroup, updateSite bool, knownScenes []string, out chan<- models.ScrapedScene, singleSceneURL string, scraperID string, siteID string, company string, siteURL string, singeScrapeAdditionalInfo string, limitScraping bool) error { +func VRPorn(wg *sync.WaitGroup, updateSite bool, knownScenes []string, out chan<- models.ScrapedScene, singleSceneURL string, scraperID string, siteID string, company string, siteURL string, singeScrapeAdditionalInfo string, limitScraping bool, masterSiteId string) error { defer wg.Done() logScrapeStart(scraperID, siteID) @@ -37,6 +37,7 @@ func VRPorn(wg *sync.WaitGroup, updateSite bool, knownScenes []string, out chan< sc.Studio = company sc.Site = siteID sc.HomepageURL = strings.Split(e.Request.URL.String(), "?")[0] + sc.MasterSiteId = masterSiteId if scraperID == "" { // there maybe no site/studio if user is jusy scraping a scene url @@ -150,7 +151,7 @@ func VRPorn(wg *sync.WaitGroup, updateSite bool, knownScenes []string, out chan< siteCollector.OnHTML(`body.tax-studio article.post div.tube-post a`, func(e *colly.HTMLElement) { sceneURL := e.Request.AbsoluteURL(e.Attr("href")) - // If scene exists in database, there's no need to scrape + // If scene exists in database, or the slternate source exists, there's no need to scrape if !funk.ContainsString(knownScenes, sceneURL) { sceneCollector.Visit(sceneURL) } @@ -169,7 +170,7 @@ func VRPorn(wg *sync.WaitGroup, updateSite bool, knownScenes []string, out chan< return nil } -func addVRPornScraper(id string, name string, company string, avatarURL string, custom bool, siteURL string) { +func addVRPornScraper(id string, name string, company string, avatarURL string, custom bool, siteURL string, masterSiteId string) { suffixedName := name siteNameSuffix := name if custom { @@ -178,22 +179,28 @@ func addVRPornScraper(id string, name string, company string, avatarURL string, } else { suffixedName += " (VRPorn)" } - registerScraper(id, suffixedName, avatarURL, "vrporn.com", func(wg *sync.WaitGroup, updateSite bool, knownScenes []string, out chan<- models.ScrapedScene, singleSceneURL string, singeScrapeAdditionalInfo string, limitScraping bool) error { - return VRPorn(wg, updateSite, knownScenes, out, singleSceneURL, id, siteNameSuffix, company, siteURL, singeScrapeAdditionalInfo, limitScraping) - }) + if masterSiteId == "" { + registerScraper(id, suffixedName, avatarURL, "vrporn.com", func(wg *sync.WaitGroup, updateSite bool, knownScenes []string, out chan<- models.ScrapedScene, singleSceneURL string, singeScrapeAdditionalInfo string, limitScraping bool) error { + return VRPorn(wg, updateSite, knownScenes, out, singleSceneURL, id, siteNameSuffix, company, siteURL, singeScrapeAdditionalInfo, limitScraping, "") + }) + } else { + registerAlternateScraper(id, suffixedName, avatarURL, "vrporn.com", masterSiteId, func(wg *sync.WaitGroup, updateSite bool, knownScenes []string, out chan<- models.ScrapedScene, singleSceneURL string, singeScrapeAdditionalInfo string, limitScraping bool) error { + return VRPorn(wg, updateSite, knownScenes, out, singleSceneURL, id, siteNameSuffix, company, siteURL, singeScrapeAdditionalInfo, limitScraping, masterSiteId) + }) + } } func init() { registerScraper("vrporn-single_scene", "VRPorn - Other Studios", "", "vrporn.com", func(wg *sync.WaitGroup, updateSite bool, knownScenes []string, out chan<- models.ScrapedScene, singleSceneURL string, singeScrapeAdditionalInfo string, limitScraping bool) error { - return VRPorn(wg, updateSite, knownScenes, out, singleSceneURL, "", "", "", "", singeScrapeAdditionalInfo, limitScraping) + return VRPorn(wg, updateSite, knownScenes, out, singleSceneURL, "", "", "", "", singeScrapeAdditionalInfo, limitScraping, "") }) var scrapers config.ScraperList scrapers.Load() for _, scraper := range scrapers.XbvrScrapers.VrpornScrapers { - addVRPornScraper(scraper.ID, scraper.Name, scraper.Company, scraper.AvatarUrl, false, scraper.URL) + addVRPornScraper(scraper.ID, scraper.Name, scraper.Company, scraper.AvatarUrl, false, scraper.URL, scraper.MasterSiteId) } for _, scraper := range scrapers.CustomScrapers.VrpornScrapers { - addVRPornScraper(scraper.ID, scraper.Name, scraper.Company, scraper.AvatarUrl, true, scraper.URL) + addVRPornScraper(scraper.ID, scraper.Name, scraper.Company, scraper.AvatarUrl, true, scraper.URL, scraper.MasterSiteId) } } diff --git a/pkg/server/cron.go b/pkg/server/cron.go index 4be4407ec..001fd1964 100644 --- a/pkg/server/cron.go +++ b/pkg/server/cron.go @@ -17,6 +17,7 @@ var rescanTask cron.EntryID var previewTask cron.EntryID var actorScrapeTask cron.EntryID var stashdbScrapeTask cron.EntryID +var linkScenesTask cron.EntryID func SetupCron() { cronInstance = cron.New() @@ -43,6 +44,10 @@ func SetupCron() { log.Println(fmt.Sprintf("Setup Stashdb Rescrape Task %v", formatCronSchedule(config.CronSchedule(config.Config.Cron.StashdbRescrapeSchedule)))) stashdbScrapeTask, _ = cronInstance.AddFunc(formatCronSchedule(config.CronSchedule(config.Config.Cron.StashdbRescrapeSchedule)), stashdbRescrapeCron) } + if config.Config.Cron.LinkScenesSchedule.Enabled { + log.Println(fmt.Sprintf("Setup Link Scenes Task %v", formatCronSchedule(config.CronSchedule(config.Config.Cron.LinkScenesSchedule)))) + linkScenesTask, _ = cronInstance.AddFunc(formatCronSchedule(config.CronSchedule(config.Config.Cron.LinkScenesSchedule)), linkScenesCron) + } cronInstance.Start() go tasks.CalculateCacheSizes() @@ -62,12 +67,16 @@ func SetupCron() { if config.Config.Cron.StashdbRescrapeSchedule.RunAtStartDelay > 0 { time.AfterFunc(time.Duration(config.Config.Cron.StashdbRescrapeSchedule.RunAtStartDelay)*time.Minute, stashdbRescrapeCron) } + if config.Config.Cron.LinkScenesSchedule.RunAtStartDelay > 0 { + time.AfterFunc(time.Duration(config.Config.Cron.LinkScenesSchedule.RunAtStartDelay)*time.Minute, linkScenesCron) + } log.Println(fmt.Sprintf("Next Rescrape Task at %v", cronInstance.Entry(rescrapTask).Next)) log.Println(fmt.Sprintf("Next Rescan Task at %v", cronInstance.Entry(rescanTask).Next)) log.Println(fmt.Sprintf("Next Preview Generation Task at %v", cronInstance.Entry(previewTask).Next)) log.Println(fmt.Sprintf("Next Actor Rescripe Task at %v", cronInstance.Entry(actorScrapeTask).Next)) log.Println(fmt.Sprintf("Next Stashdb Rescrape Task at %v", cronInstance.Entry(stashdbScrapeTask).Next)) + log.Println(fmt.Sprintf("Next Link Scenes Task at %v", cronInstance.Entry(linkScenesTask).Next)) } func scrapeCron() { @@ -96,6 +105,13 @@ func stashdbRescrapeCron() { log.Println(fmt.Sprintf("Next Stashdb Rescrape Task at %v", cronInstance.Entry(rescrapTask).Next)) } +func linkScenesCron() { + if !session.HasActiveSession() { + tasks.MatchAlternateSources() + } + log.Println(fmt.Sprintf("Next Link Scenes Task at %v", cronInstance.Entry(rescrapTask).Next)) +} + var previewGenerateInProgress = false func generatePreviewCron() { diff --git a/pkg/tasks/alternate_scene_source.go b/pkg/tasks/alternate_scene_source.go new file mode 100644 index 000000000..6760dc817 --- /dev/null +++ b/pkg/tasks/alternate_scene_source.go @@ -0,0 +1,280 @@ +package tasks + +import ( + "encoding/json" + "fmt" + "strings" + "time" + + "github.com/blevesearch/bleve/v2" + "github.com/jinzhu/gorm" + "github.com/xbapps/xbvr/pkg/config" + "github.com/xbapps/xbvr/pkg/models" +) + +func AddAlternateSceneSource(db *gorm.DB, scrapedScene models.ScrapedScene) { + var extref models.ExternalReference + source := "alternate scene " + scrapedScene.ScraperID + extref.FindExternalId(source, scrapedScene.SceneID) + extref.ExternalId = scrapedScene.SceneID + extref.ExternalSource = source + extref.ExternalURL = scrapedScene.HomepageURL + + var scene models.Scene + scene.PopulateSceneFieldsFromExternal(db, scrapedScene) + extref.ExternalDate = scene.ReleaseDate + + // strip out other actor columns, it makes the data too large and we only need the name + var newCastList []models.Actor + for _, actor := range scene.Cast { + newCastList = append(newCastList, models.Actor{ID: actor.ID, Name: actor.Name}) + } + scene.Cast = newCastList + var data models.SceneAlternateSource + data.MasterSiteId = scrapedScene.MasterSiteId + data.Scene = scene + data.MatchAchieved = -1 + jsonData, _ := json.Marshal(data) + extref.ExternalData = string(jsonData) + extref.UdfBool1 = scene.HumanScript + extref.UdfBool2 = scene.AiScript + extref.UdfDatetime1 = scene.ScriptPublished + extref.Save() +} + +func MatchAlternateSources() { + tlog := log.WithField("task", "scrape") + tlog.Info("Matching scenes from alternate sources") + commonDb, _ := models.GetCommonDB() + + var unmatchedScenes []models.ExternalReference + + commonDb.Joins("Left JOIN external_reference_links erl on erl.external_reference_id = external_references.id"). + Where("external_references.external_source like 'alternate scene %' and erl.external_reference_id is NULL"). + Find(&unmatchedScenes) + + // check for scenes that should be relinked based on the reprocess_links param + var sites []models.Site + commonDb.Where("matching_params<> '' and JSON_EXTRACT(matching_params, '$.reprocess_links')>0").Find(&sites) + for _, site := range sites { + var reprocessList []models.ExternalReference + var matchParams models.AltSrcMatchParams + matchParams.UnmarshalParams(site.MatchingParams) + reprocessDate := time.Now().AddDate(0, 0, matchParams.ReprocessLinks*-1) + // ignore match_type 99999, they are manually match and shouldn't change + commonDb.Preload("XbvrLinks"). + Joins("Join external_reference_links on external_reference_links.external_reference_id = external_references.id"). + Where("external_references.external_source = ? and external_references.external_date > ? and external_reference_links.match_type not in (-1, 99999)", "alternate scene "+site.ID, reprocessDate).Find(&reprocessList) + unmatchedScenes = append(unmatchedScenes, reprocessList...) + + } + lastProgressUpdate := time.Now() + for cnt, altsource := range unmatchedScenes { + if time.Since(lastProgressUpdate) > time.Duration(config.Config.Advanced.ProgressTimeInterval)*time.Second { + tlog.Infof("Matching alternate scene sources %v of %v", cnt, len(unmatchedScenes)) + lastProgressUpdate = time.Now() + } + var unmatchedSceneData models.SceneAlternateSource + var possiblematchs []models.Scene + var site models.Site + var matchParams models.AltSrcMatchParams + json.Unmarshal([]byte(altsource.ExternalData), &unmatchedSceneData) + site.GetIfExist(strings.TrimPrefix(altsource.ExternalSource, "alternate scene ")) + err := matchParams.UnmarshalParams(site.MatchingParams) + if err != nil { + tlog.Infof("Cannot read Matching parameters for %s %s", site.Name, err) + continue + } + if matchParams.DelayLinking > 0 { + skipDate := altsource.ExternalDate.AddDate(0, 0, matchParams.DelayLinking) + if skipDate.After(time.Now()) { + continue + } + } + + if !matchParams.IgnoreReleasedBefore.IsZero() { + if unmatchedSceneData.Scene.ReleaseDate.Before(matchParams.IgnoreReleasedBefore) { + continue + } + } else { + if !config.Config.Advanced.IgnoreReleasedBefore.IsZero() { + if unmatchedSceneData.Scene.ReleaseDate.Before(config.Config.Advanced.IgnoreReleasedBefore) { + continue + } + } + } + + tmpTitle := strings.ReplaceAll(strings.ReplaceAll(unmatchedSceneData.Scene.Title, " and ", "&"), "+", "&") + for _, char := range `'- :!?"/.,@()–` { + tmpTitle = strings.ReplaceAll(tmpTitle, string(char), "") + } + commonDb.Preload("Cast").Where(`replace(replace(replace(replace(replace(replace(replace(replace(replace(replace(replace(replace(replace(replace(replace(replace(title,'''',''),'-',''),' ',''),':',''),'!',''),'?',''),'"',''),'/',''),'.',''),',',''),' and ','&'),'@',''),')',''),'(',''), '+','&'),'–','') like ? and scraper_id = ?`, + tmpTitle, unmatchedSceneData.MasterSiteId).Find(&possiblematchs) + + if len(possiblematchs) == 1 { + found := false + // check not already linked, if we update unnessarily, it will mess up the sort by "Released on Alternate Sites" sort option + for _, link := range altsource.XbvrLinks { + if link.InternalDbId == possiblematchs[0].ID { + found = true + break + } + } + if !found { + UpdateLinks(commonDb, altsource.ID, models.ExternalReferenceLink{InternalTable: "scenes", InternalDbId: possiblematchs[0].ID, InternalNameId: possiblematchs[0].SceneID, + ExternalReferenceID: altsource.ID, ExternalSource: altsource.ExternalSource, ExternalId: altsource.ExternalId, MatchType: 10000, UdfDatetime1: time.Now()}) + } + } else { + // build the search query based on the sites matching params + ignoreChars := `+-=&|> 0 { + sitename = masterSite.Name[:strings.Index(masterSite.Name, `(`)-1] + } + + q := fmt.Sprintf(`+site:"%s" title:"%s"^%v`, sitename, title, matchParams.BoostTitleExact) + for _, word := range strings.Fields(title) { + q = fmt.Sprintf(`%s title:%s^%v`, q, word, matchParams.BoostTitleAnyWords) + } + + if matchParams.CastMatchType == "should" { + for _, actor := range unmatchedSceneData.Scene.Cast { + q = fmt.Sprintf(`%s cast:"%s"^%v`, q, actor.Name, matchParams.BoostCast) + // split to indivual words as well, eg handles cases like "someone baberoticvr" or one site only using a first name + // but lower the boosting a bit + words := strings.Fields(actor.Name) + for _, word := range words { + q = fmt.Sprintf(`%s cast:%s^%0.2f`, q, word, matchParams.BoostCast*0.75) + } + } + } + + if matchParams.DurationMatchType != "do not" { + // note: no boosting for duration, doesn't appear to work except on ranges, where it must be within the range anyway + if matchParams.DurationMatchType == "must" { + matchParams.DurationRangeLess = 0 + matchParams.DurationRangeMore = 0 + } + if unmatchedSceneData.Scene.Duration >= matchParams.DurationMin { + if matchParams.DurationRangeLess+matchParams.DurationRangeMore > 1 { + q = fmt.Sprintf("%s duration: %v duration:>=%v duration:<=%v", q, unmatchedSceneData.Scene.Duration, unmatchedSceneData.Scene.Duration-matchParams.DurationRangeLess, unmatchedSceneData.Scene.Duration+matchParams.DurationRangeMore) + } else { + q = fmt.Sprintf("%s duration: %v", q, unmatchedSceneData.Scene.Duration) + } + } + } + + if matchParams.ReleasedMatchType != "do not" { + if matchParams.ReleasedMatchType == "must" { + matchParams.ReleasedPrior = 0 + matchParams.ReleasedAfter = 0 + } + if matchParams.IgnoreReleasedBefore.IsZero() || unmatchedSceneData.Scene.ReleaseDate.After(matchParams.IgnoreReleasedBefore.AddDate(0, 0, -1)) { + prefix := "" + if matchParams.ReleasedMatchType == "must" { + prefix = "+" + } + q = fmt.Sprintf(`%s %sreleased:>="%s"^%v %sreleased:<="%s"^%v`, q, + prefix, unmatchedSceneData.Scene.ReleaseDate.AddDate(0, 0, matchParams.ReleasedPrior*-1).Format("2006-01-02"), matchParams.BoostReleased, + prefix, unmatchedSceneData.Scene.ReleaseDate.AddDate(0, 0, matchParams.ReleasedAfter).Format("2006-01-02"), matchParams.BoostReleased) + } + } + + if matchParams.DescriptionMatchType == "should" { // "must" is not an option for description + words := strings.Fields(desc) + for _, word := range words { + if strings.TrimSpace(word) != "" { + q = fmt.Sprintf("%s description: %v^%v", q, strings.TrimSpace(word), matchParams.BoostDescription) + } + } + } + + query := bleve.NewQueryStringQuery(q) + + searchRequest := bleve.NewSearchRequest(query) + searchRequest.Fields = []string{"title", "cast", "site", "description", "released"} + searchRequest.IncludeLocations = true + searchRequest.From = 0 + searchRequest.Size = 25 + searchRequest.SortBy([]string{"-_score"}) + + searchResults, err := AltSourceSearch(searchRequest) + if err != nil { + log.Error(err) + log.Error(q) + continue + } + + var extdata models.SceneAlternateSource + // remove saving the query later, keep for debug purposes + json.Unmarshal([]byte(altsource.ExternalData), &extdata) + extdata.Query = q + newjson, _ := json.Marshal(extdata) + altsource.ExternalData = string(newjson) + tmpAltSource := altsource + tmpAltSource.XbvrLinks = nil + // save the updated query used, but don't update the links + tmpAltSource.Save() + if len(searchResults.Hits) > 0 { + var scene models.Scene + scene.GetIfExist(searchResults.Hits[0].ID) + if scene.ID > 0 { + // check not already linked, if we update unnessarily, it will mess up the sort by "Released on Alternate Sites" sort option + found := false + for _, link := range altsource.XbvrLinks { + if link.InternalDbId == scene.ID { + found = true + break + } + } + if !found { + UpdateLinks(commonDb, altsource.ID, models.ExternalReferenceLink{InternalTable: "scenes", InternalDbId: scene.ID, InternalNameId: scene.SceneID, + ExternalReferenceID: altsource.ID, ExternalSource: altsource.ExternalSource, ExternalId: altsource.ExternalId, MatchType: int(searchResults.Hits[0].Score), UdfDatetime1: time.Now()}) + } + } + } + } + } + tlog.Info("Completed Matching scenes from alternate sources") +} +func AltSourceSearch(searchRequest *bleve.SearchRequest) (*bleve.SearchResult, error) { + // open and close the search for each search, this stops the search function from locking users out of searching + idx, err := NewIndex("scenes") + if err != nil { + return nil, err + } + defer idx.Bleve.Close() + return idx.Bleve.Search(searchRequest) +} +func UpdateLinks(db *gorm.DB, externalreference_id uint, newLink models.ExternalReferenceLink) { + var extref models.ExternalReference + extref.GetIfExist(externalreference_id) + exists := false + + db.Where("external_source = ? and external_reference_id = ? and internal_db_id <> ?").Find(&extref) + for _, link := range extref.XbvrLinks { + if link.InternalDbId == newLink.InternalDbId { + exists = true + } else { + link.Delete() + } + } + if !exists { + newLink.Save() + } + +} diff --git a/pkg/tasks/content.go b/pkg/tasks/content.go index bb32fb22e..3a7e920ff 100644 --- a/pkg/tasks/content.go +++ b/pkg/tasks/content.go @@ -55,22 +55,23 @@ type BackupActionActor struct { ActionActors []models.ActionActor `xbvrbackup:"action_actors"` } type BackupContentBundle struct { - Timestamp time.Time `xbvrbackup:"timestamp"` - BundleVersion string `xbvrbackup:"bundleVersion"` - Volumne []models.Volume `xbvrbackup:"volumes"` - Playlists []models.Playlist `xbvrbackup:"playlists"` - Sites []models.Site `xbvrbackup:"sites"` - Scenes []models.Scene `xbvrbackup:"scenes"` - FilesLinks []BackupFileLink `xbvrbackup:"sceneFileLinks"` - Cuepoints []BackupSceneCuepoint `xbvrbackup:"sceneCuepoints"` - History []BackupSceneHistory `xbvrbackup:"sceneHistory"` - Actions []BackupSceneAction `xbvrbackup:"actions"` - Akas []models.Aka `xbvrbackup:"akas"` - TagGroups []models.TagGroup `xbvrbackup:"tagGroups"` - ExternalRefs []models.ExternalReference `xbvrbackup:"externalReferences"` - Actors []models.Actor `xbvrbackup:"actors"` - ActionActors []BackupActionActor `xbvrbackup:"actionActors"` - Kvs []models.KV `xbvrbackup:"config"` + Timestamp time.Time `xbvrbackup:"timestamp"` + BundleVersion string `xbvrbackup:"bundleVersion"` + Volumne []models.Volume `xbvrbackup:"volumes"` + Playlists []models.Playlist `xbvrbackup:"playlists"` + Sites []models.Site `xbvrbackup:"sites"` + Scenes []models.Scene `xbvrbackup:"scenes"` + FilesLinks []BackupFileLink `xbvrbackup:"sceneFileLinks"` + Cuepoints []BackupSceneCuepoint `xbvrbackup:"sceneCuepoints"` + History []BackupSceneHistory `xbvrbackup:"sceneHistory"` + Actions []BackupSceneAction `xbvrbackup:"actions"` + Akas []models.Aka `xbvrbackup:"akas"` + TagGroups []models.TagGroup `xbvrbackup:"tagGroups"` + ExternalRefs []models.ExternalReference `xbvrbackup:"externalReferences"` + ManualSceneMatches []models.ExternalReference `xbvrbackup:"manualSceneMatches"` + Actors []models.Actor `xbvrbackup:"actors"` + ActionActors []BackupActionActor `xbvrbackup:"actionActors"` + Kvs []models.KV `xbvrbackup:"config"` } type RequestRestore struct { InclAllSites bool `json:"allSites"` @@ -91,6 +92,7 @@ type RequestRestore struct { InclActors bool `json:"inclActors"` InclActorActions bool `json:"inclActorActions"` InclConfig bool `json:"inclConfig"` + ExtRefSubset string `json:"extRefSubset"` } func CleanTags() { @@ -160,14 +162,19 @@ func sceneDBWriter(wg *sync.WaitGroup, i *uint64, scenes <-chan models.ScrapedSc models.SceneUpdateScriptData(commonDb, scene) } } else { - models.SceneCreateUpdateFromExternal(commonDb, scene) + if scene.MasterSiteId == "" { + models.SceneCreateUpdateFromExternal(commonDb, scene) + } else { + AddAlternateSceneSource(commonDb, scene) + } + } + if scene.MasterSiteId == "" { + // Add the processed scene to the list to re/index + lock.Lock() + *processedScenes = append(*processedScenes, scene) + lock.Unlock() + atomic.AddUint64(i, 1) } - // Add the processed scene to the list to re/index - lock.Lock() - *processedScenes = append(*processedScenes, scene) - lock.Unlock() - - atomic.AddUint64(i, 1) if os.Getenv("DEBUG") != "" { log.Printf("Saved %v", scene.SceneID) } @@ -283,8 +290,10 @@ func Scrape(toScrape string, singleSceneURL string, singeScrapeAdditionalInfo st // Get all known scenes var scenes []models.Scene + var extrefs []models.ExternalReference commonDb, _ := models.GetCommonDB() commonDb.Find(&scenes) + commonDb.Where("external_source like 'alternate scene %'").Find(&extrefs) var knownScenes []string for i := range scenes { @@ -292,6 +301,9 @@ func Scrape(toScrape string, singleSceneURL string, singeScrapeAdditionalInfo st knownScenes = append(knownScenes, scenes[i].SceneURL) } } + for i := range extrefs { + knownScenes = append(knownScenes, extrefs[i].ExternalURL) + } collectedScenes := make(chan models.ScrapedScene, 250) var sceneCount uint64 @@ -334,6 +346,9 @@ func Scrape(toScrape string, singleSceneURL string, singeScrapeAdditionalInfo st ReapplyEdits() IndexScrapedScenes(&processedScenes) + if config.Config.Advanced.LinkScenesAfterSceneScraping { + MatchAlternateSources() + } tlog.Infof("Scraped %v new scenes in %s", sceneCount, @@ -518,7 +533,7 @@ func ImportBundleV1(bundleData ContentBundle) { } -func BackupBundle(inclAllSites bool, onlyIncludeOfficalSites bool, inclScenes bool, inclFileLinks bool, inclCuepoints bool, inclHistory bool, inclPlaylists bool, InclActorAkas bool, inclTagGroups bool, inclVolumes bool, inclSites bool, inclActions bool, inclExtRefs bool, inclActors bool, inclActorActions bool, inclConfig bool, playlistId string, outputBundleFilename string, version string) string { +func BackupBundle(inclAllSites bool, onlyIncludeOfficalSites bool, inclScenes bool, inclFileLinks bool, inclCuepoints bool, inclHistory bool, inclPlaylists bool, InclActorAkas bool, inclTagGroups bool, inclVolumes bool, inclSites bool, inclActions bool, inclExtRefs bool, inclActors bool, inclActorActions bool, inclConfig bool, extRefSubset string, playlistId string, outputBundleFilename string, version string) string { var out BackupContentBundle var content []byte exportCnt := 0 @@ -660,9 +675,15 @@ func BackupBundle(inclAllSites bool, onlyIncludeOfficalSites bool, inclScenes bo } var externalReferences []models.ExternalReference + var filteredxternalReferences []models.ExternalReference if inclExtRefs { lastMessage := time.Now() - db.Order("external_source").Order("id").Find(&externalReferences) + switch extRefSubset { + case "": + db.Order("external_source").Order("external_source").Order("external_id").Find(&externalReferences) + case "manual_matched", "deleted_match": + db.Where("external_source like 'alternate scene %'").Order("external_source").Order("external_id").Find(&externalReferences) + } recCnt := 0 for idx, ref := range externalReferences { if time.Since(lastMessage) > time.Duration(config.Config.Advanced.ProgressTimeInterval)*time.Second { @@ -670,10 +691,28 @@ func BackupBundle(inclAllSites bool, onlyIncludeOfficalSites bool, inclScenes bo lastMessage = time.Now() } var links []models.ExternalReferenceLink - db.Where("external_reference_id = ?", ref.ID).Find(&links) - externalReferences[idx].XbvrLinks = links + switch extRefSubset { + case "", "all": + db.Where("external_reference_id = ?", ref.ID).Order("external_source").Order("external_id").Find(&links) + externalReferences[idx].XbvrLinks = links + case "manual_matched": + db.Where("external_reference_id = ? and match_type=99999", ref.ID).Order("external_source").Order("external_id").Find(&links) + if len(links) > 0 { + externalReferences[idx].XbvrLinks = links + filteredxternalReferences = append(filteredxternalReferences, externalReferences[idx]) + } + case "deleted_match": + db.Where("external_reference_id = ? and match_type=-1 and internal_name_id='deleted'", ref.ID).Order("external_source").Order("external_id").Find(&links) + if len(links) > 0 { + externalReferences[idx].XbvrLinks = links + filteredxternalReferences = append(filteredxternalReferences, externalReferences[idx]) + } + } recCnt += 1 } + if extRefSubset != "" { + externalReferences = filteredxternalReferences + } tlog.Infof("Reading %v of %v external references", recCnt, len(externalReferences)) } @@ -839,7 +878,7 @@ func RestoreBundle(request RequestRestore) { tagGroup.UpdateSceneTagRecords() } if request.InclExternalRefs { - RestoreExternalRefs(bundleData.ExternalRefs, request.Overwrite, db) + RestoreExternalRefs(bundleData.ExternalRefs, request.Overwrite, request.ExtRefSubset, db) } if request.InclActors { RestoreActors(bundleData.Actors, request.Overwrite, db) @@ -1310,7 +1349,7 @@ func RenameTags() { } } -func RestoreExternalRefs(extRefs []models.ExternalReference, overwrite bool, db *gorm.DB) { +func RestoreExternalRefs(extRefs []models.ExternalReference, overwrite bool, extRefSubset string, db *gorm.DB) { tlog := log.WithField("task", "scrape") tlog.Infof("Restoring External References") @@ -1322,6 +1361,35 @@ func RestoreExternalRefs(extRefs []models.ExternalReference, overwrite bool, db lastMessage = time.Now() } + // if we specified a filter check if we should import + switch extRefSubset { + case "manual_matched": + if !strings.HasPrefix(extRef.ExternalSource, "alternate scene") { + continue + } + skip := true + for _, link := range extRef.XbvrLinks { + if link.MatchType == 99999 { + skip = false + } + } + if skip { + continue + } + case "deleted_match": + if !strings.HasPrefix(extRef.ExternalSource, "alternate scene") { + continue + } + skip := true + for _, link := range extRef.XbvrLinks { + if link.MatchType == -1 && link.InternalNameId == "deleted" { + skip = false + } + } + if skip { + continue + } + } var found models.ExternalReference db.Preload("XbvrLinks").Where(&models.ExternalReference{ExternalSource: extRef.ExternalSource, ExternalId: extRef.ExternalId}).First(&found) @@ -1341,12 +1409,15 @@ func RestoreExternalRefs(extRefs []models.ExternalReference, overwrite bool, db // case "sites": // extRef.XbvrLinks[idx].InternalDbId = link.InternalNameId case "scenes": - var scene models.Scene - scene.GetIfExist(link.InternalNameId) - if scene.ID == 0 { - continue + if link.InternalNameId != "deleted" { + // if the name is deleted, we don't need the scene id + var scene models.Scene + scene.GetIfExist(link.InternalNameId) + if scene.ID == 0 { + continue + } + extRef.XbvrLinks[idx].InternalDbId = scene.ID } - extRef.XbvrLinks[idx].InternalDbId = scene.ID case "actors": var actor models.Actor db.Where("name = ?", link.InternalNameId).First(&actor) diff --git a/pkg/tasks/volume.go b/pkg/tasks/volume.go index c4a0d104b..4c9def118 100644 --- a/pkg/tasks/volume.go +++ b/pkg/tasks/volume.go @@ -61,6 +61,7 @@ func RescanVolumes(id int) { // Match Scene to File var files []models.File var scenes []models.Scene + var extrefs []models.ExternalReference tlog.Infof("Matching Scenes to known filenames") db.Model(&models.File{}).Where("files.scene_id = 0").Find(&files) @@ -81,7 +82,31 @@ func RescanVolumes(id int) { if err != nil { log.Error(err, " when matching "+unescapedFilename) } - + if len(scenes) == 0 && config.Config.Advanced.UseAltSrcInFileMatching { + // check if the filename matches in external_reference record + + db.Preload("XbvrLinks").Where("external_source like 'alternate scene %' and external_data LIKE ? OR external_data LIKE ? OR external_data LIKE ?", `%"`+filename+`%`, `%"`+filename2+`%`, `%"`+filename3+`%`).Find(&extrefs) + if len(extrefs) == 1 { + if len(extrefs[0].XbvrLinks) == 1 { + // the scene id will be the Internal DB Id from the associated link + var scene models.Scene + scene.GetIfExistByPK(extrefs[0].XbvrLinks[0].InternalDbId) + // Add File to the list of Scene filenames + var pfTxt []string + err = json.Unmarshal([]byte(scene.FilenamesArr), &pfTxt) + if err != nil { + continue + } + pfTxt = append(pfTxt, files[i].Filename) + tmp, err := json.Marshal(pfTxt) + if err == nil { + scene.FilenamesArr = string(tmp) + } + scene.Save() + scenes = append(scenes, scene) + } + } + } if len(scenes) == 1 { files[i].SceneID = scenes[0].ID files[i].Save() diff --git a/ui/src/QuickFind.vue b/ui/src/QuickFind.vue index 2b22d6d1d..17ea9ff6b 100644 --- a/ui/src/QuickFind.vue +++ b/ui/src/QuickFind.vue @@ -87,16 +87,23 @@ export default { computed: { isActive: { get () { - const out = this.$store.state.overlay.showQuickFind + if (this.queryString!=null && this.queryString!="") { + this.getAsyncData(this.queryString) + } + const out = this.$store.state.overlay.quickFind.show if (out === true) { this.$nextTick(() => { this.$refs.autocompleteInput.$refs.input.focus() + if (this.$store.state.overlay.quickFind.searchString !=null && this.$store.state.overlay.quickFind.searchString!="") { + this.queryString = this.$store.state.overlay.quickFind.searchString + this.$store.state.overlay.quickFind.searchString = null + } }) } return out }, set (values) { - this.$store.state.overlay.showQuickFind = values + this.$store.state.overlay.quickFind.show = values } } }, @@ -153,12 +160,20 @@ export default { } }, showSceneDetails (scene) { - if (this.$router.currentRoute.name !== 'scenes') { - this.$router.push({ name: 'scenes' }) - } this.$store.commit('overlay/hideQuickFind') - this.data = [] - this.$store.commit('overlay/showDetails', { scene }) + if (this.$store.state.overlay.quickFind.displaySelectedScene) { + if (this.$router.currentRoute.name !== 'scenes') { + this.$router.push({ name: 'scenes' }) + } + this.$store.commit('overlay/hideQuickFind') + this.data = [] + this.$store.commit('overlay/showDetails', { scene }) + } else { + // don't display the scene, just pass the selected scene back in the $store.state and close + this.$store.state.overlay.quickFind.selectedScene = scene + this.$store.commit('overlay/hideQuickFind') + this.data = [] + } }, searchPrefix(prefix) { let textbox = this.$refs.autocompleteInput.$refs.input.$refs.input diff --git a/ui/src/components/EditButton.vue b/ui/src/components/EditButton.vue index 1066328c2..fa843e355 100644 --- a/ui/src/components/EditButton.vue +++ b/ui/src/components/EditButton.vue @@ -1,7 +1,7 @@ diff --git a/ui/src/locales/en-GB.json b/ui/src/locales/en-GB.json index c548dfac4..f77d47886 100644 --- a/ui/src/locales/en-GB.json +++ b/ui/src/locales/en-GB.json @@ -207,5 +207,37 @@ "Occasionaly test uploading your export bundles. Browser memory constraints may cause problems restoring large exports. Use this function to test if your browser can load an export.": "Occasionaly test uploading your export bundles. Browser memory constraints may cause problems restoring large exports. Use this function to test if your browser can load an export.", "Only the first page of scenes will be scraped":"Only the first page of scenes will be scraped", "Limit Scraping":"Limit Scraping", + "Leave blank, unless you want to link scenes from the new custom site to scenes on an existing studio site, e.g. VRHush on SLR or VRPorn to the main VRHush site": "Leave blank, unless you want to link scenes from the new custom site to scenes on an existing studio site, e.g. VRHush on SLR or VRPorn to the main VRHush site", + "Display scene details":"Display scene details", + "Matching paramerters":"Matching paramerters", + "Days to wait after the release date, before linking. Useful where the main site releases after SLR/VRPorn/POVR, eg LethalHardware": "Days to wait after the release date, before linking. Useful where the main site releases after SLR/VRPorn/POVR, eg LethalHardware", + "Number of days to keep re-linking scenes after the release date":"Number of days to keep re-linking scenes after the release date", + "Keep Re-linking(days)":"Keep Re-linking(days)", + "Delay linking(days)":"Delay linking(days)", + "Do not link scenes prior to the specified date. The quality of metadata of older scenes is often poor and causes mismatches":"Do not link scenes prior to the specified date. The quality of metadata of older scenes is often poor and causes mismatches", + "Ignore Scenes Released Prior To":"Ignore Scenes Released Prior To", + "Weighting of Title matchs (vs Duration=1)":"Weighting of Title matchs (vs Duration=1)", + "Boost Value":"Boost Value", + "The number of days prior to the release date to match, eg if the scene release date is 23/05/2023 and the days prior is 3, it will search >= 20/05/2023. If days prior and after are 0, the range is not used":"The number of days prior to the release date to match, eg if the scene release date is 23/05/2023 and the days prior is 3, it will search >= 20/05/2023. If days prior and after are 0, the range is not used", + "Days Prior":"Days Prior", + "The number of days after the release date to match, eg if the scene release date is 23/05/2023 and the days after is =3, it will search <= 23/05/2023. Usually set to 0. If days prior and after are 0, the range is not used":"The number of days after the release date to match, eg if the scene release date is 23/05/2023 and the days after is =3, it will search <= 23/05/2023. Usually set to 0. If days prior and after are 0, the range is not used", + "Days After":"Days After", + "Exact Match Boost Value":"Exact Match Boost Value", + "Individual Word Match Boost Value":"Individual Word Match Boost Value", + "Match Type":"Match Type", + "Minimum Duration":"Minimum Duration", + "Lower Search Range":"Lower Search Range", + "Upper Search Range":"Upper Search Range", + "Alternate Sites":"Alternate Sites", + "Main Site":"Main Site", + "Scenes from Akternate Sites will be matched after Scene Scraping":"Scenes from Akternate Sites will be matched after Scene Scraping", + "If a file is not matched to a scene, then try scenes from Alternate Sites":"If a file is not matched to a scene, then try scenes from Alternate Sites", + "When filtering for Scenes with Scripts or sorting by Script Published Date, scenes from Alternate Sites will be included. Note: Slows these queries":"When filtering for Scenes with Scripts or sorting by Script Published Date, scenes from Alternate Sites will be included. Note: Slows these queries", + "Do not link scenes prior to the specified date. The quality of metadata of older scenes is often poor and causes mismatches":"Do not link scenes prior to the specified date. The quality of metadata of older scenes is often poor and causes mismatches", + "Remove Scene Links":"Remove Scene Links", + "Remove Scene Links (Keep edits)":"Remove Scene Links (Keep edits)", + "Delete scraped scenes":"Delete scraped scenes", + "Link Scenes":"Link Scenes", + "Linked to Alternate Sites":"Linked to Alternate Sites", "Go": "Go" } diff --git a/ui/src/store/index.js b/ui/src/store/index.js index 5ec5e0c14..1a7840cd5 100644 --- a/ui/src/store/index.js +++ b/ui/src/store/index.js @@ -16,6 +16,7 @@ import optionsPreviews from './optionsPreviews' import optionsFunscripts from './optionsFunscripts' import optionsVendor from './optionsVendor' import optionsAdvanced from './optionsAdvanced' +import optionsSceneCreate from './optionsSceneCreate' Vue.use(Vuex) @@ -35,6 +36,7 @@ export default new Vuex.Store({ optionsPreviews, optionsFunscripts, optionsVendor, - optionsAdvanced + optionsAdvanced, + optionsSceneCreate, } }) diff --git a/ui/src/store/optionsAdvanced.js b/ui/src/store/optionsAdvanced.js index 383a25831..971252bd2 100644 --- a/ui/src/store/optionsAdvanced.js +++ b/ui/src/store/optionsAdvanced.js @@ -9,6 +9,10 @@ const state = { stashApiKey: '', scrapeActorAfterScene: 'true', useImperialEntry: 'false', + linkScenesAfterSceneScraping: true, + useAltSrcInFileMatching: true, + useAltSrcInScriptFilters: true, + ignoreReleasedBefore: null, } } @@ -26,6 +30,10 @@ const actions = { state.advanced.stashApiKey = data.config.advanced.stashApiKey state.advanced.scrapeActorAfterScene = data.config.advanced.scrapeActorAfterScene state.advanced.useImperialEntry = data.config.advanced.useImperialEntry + state.advanced.linkScenesAfterSceneScraping = data.config.advanced.linkScenesAfterSceneScraping + state.advanced.useAltSrcInFileMatching = data.config.advanced.useAltSrcInFileMatching + state.advanced.useAltSrcInScriptFilters = data.config.advanced.useAltSrcInScriptFilters + state.advanced.ignoreReleasedBefore = data.config.advanced.ignoreReleasedBefore state.loading = false }) }, @@ -40,6 +48,10 @@ const actions = { state.advanced.stashApiKey = data.stashApiKey state.advanced.scrapeActorAfterScene = data.scrapeActorAfterScene state.advanced.useImperialEntry = data.useImperialEntry + state.advanced.linkScenesAfterSceneScraping = data.linkScenesAfterSceneScraping + state.advanced.useAltSrcInFileMatching = data.useAltSrcInFileMatching + state.advanced.useAltSrcInScriptFilters = data.useAltSrcInScriptFilters + state.advanced.ignoreReleasedBefore = data.ignoreReleasedBefore state.loading = false }) } diff --git a/ui/src/store/optionsSceneCreate.js b/ui/src/store/optionsSceneCreate.js new file mode 100644 index 000000000..e7e98c350 --- /dev/null +++ b/ui/src/store/optionsSceneCreate.js @@ -0,0 +1,25 @@ +import ky from 'ky' + +const state = { + scrapeScene: '', + showSceneCreate: false, +} + +const mutations = { + setScrapeScene(state, payload) { + state.scrapeScene = payload + }, + showSceneCreate(state, payload) { + state.showSceneCreate = payload + }, +} + +const actions = { +} + +export default { + namespaced: true, + state, + mutations, + actions +} diff --git a/ui/src/store/overlay.js b/ui/src/store/overlay.js index c60cf6a73..a36a808cf 100644 --- a/ui/src/store/overlay.js +++ b/ui/src/store/overlay.js @@ -1,7 +1,10 @@ const state = { details: { show: false, - scene: null + scene: null, + altsrc: null, + prevscene: null, + query_for_altsrc: '', }, edit: { show: false, @@ -26,18 +29,40 @@ const state = { actoredit: { show: false, actor: null + }, + quickFind: { + show:false, + searchString: null, // use to preppoulate the search box + displaySelectedScene: true, + selectedScene: null, // selected scene + }, + sceneMatchParams:{ // overlay to edit matching params + show:false, + site: '', }, - showQuickFind: false } const mutations = { showDetails (state, payload) { state.details.scene = payload.scene + state.details.altsrc = payload.altsrc + state.details.prevscene = payload.prevscene + state.details.query_for_altsrc = payload.query_for_altsrc state.details.show = true }, hideDetails (state, payload) { - state.details.scene = null - state.details.show = false + if (state.details.altsrc != null) { + // if we are display scene data from another source, go back to the real scene + state.details.show = false + state.details.altsrc = null + state.details.scene = state.details.prevscene + state.details.prevscene = null + state.details.query_for_altsrc = '' + state.details.show = true + }else { + state.details.scene = null + state.details.show = false + } }, editDetails (state, payload) { state.edit.scene = payload.scene @@ -88,11 +113,28 @@ const mutations = { state.createScene.show = false }, showQuickFind (state, payload) { - state.showQuickFind = true + state.quickFind.displaySelectedScene = true + state.quickFind.selectedScene = null + if (payload !== undefined) { + if (payload.searchString !== undefined) { + state.quickFind.searchString = payload.searchString + } + if (payload.displaySelectedScene !== undefined) { + state.quickFind.displaySelectedScene = payload.displaySelectedScene + } + } + state.quickFind.show = true }, hideQuickFind (state, payload) { - state.showQuickFind = false - } + state.quickFind.show = false + }, + showSceneMatchParams (state, payload) { + state.sceneMatchParams.site = payload.site + state.sceneMatchParams.show = true + }, + hideSceneMatchParams (state, payload) { + state.sceneMatchParams.show = false + }, } export default { diff --git a/ui/src/views/files/SceneMatch.vue b/ui/src/views/files/SceneMatch.vue index 4253fa96f..5a09755c8 100644 --- a/ui/src/views/files/SceneMatch.vue +++ b/ui/src/views/files/SceneMatch.vue @@ -223,8 +223,7 @@ export default { return parseInt(value, radix || 10) || defaultValue || 0 }, videoFilesCount (scene) { - let count = 0 - console.log(scene) + let count = 0 scene.file.forEach(obj => { if (obj.type === 'video') { count = count + 1 diff --git a/ui/src/views/options/Options.vue b/ui/src/views/options/Options.vue index 3484d1707..ae591e296 100644 --- a/ui/src/views/options/Options.vue +++ b/ui/src/views/options/Options.vue @@ -44,6 +44,7 @@ + @@ -64,9 +65,10 @@ import Previews from './sections/Previews.vue' import Schedules from './sections/Schedules.vue' import InterfaceDeoVR from './sections/InterfaceDeoVR.vue' import InterfaceAdvanced from './sections/InterfaceAdvanced.vue' +import SceneMatchParams from './overlays/SceneMatchParams.vue' export default { - components: { Storage, SceneDataScrapers, SceneCreate, Funscripts, SceneDataImportExport, InterfaceWeb, InterfaceDLNA, InterfaceDeoVR, Cache, Previews, Schedules, InterfaceAdvanced }, + components: { Storage, SceneDataScrapers, SceneCreate, Funscripts, SceneDataImportExport, InterfaceWeb, InterfaceDLNA, InterfaceDeoVR, Cache, Previews, Schedules, InterfaceAdvanced,SceneMatchParams }, data: function () { return { active: 'storage' @@ -76,6 +78,23 @@ export default { setActive: function (e) { this.active = e } - } + }, + computed: { + showMatchParamsOverlay () { + return this.$store.state.overlay.sceneMatchParams.show + }, + showSceneCreate() { + if (this.$store.state.optionsSceneCreate.showSceneCreate){ + this.setActive('create-scene') + this.$store.commit('optionsSceneCreate/showSceneCreate', false ) + } + return this.$store.state.optionsSceneCreate.showSceneCreate; + }, + }, + watch: { + showSceneCreate(newValue, oldValue) { + // dummy watch to trigger the computed function + }, + }, } diff --git a/ui/src/views/options/overlays/SceneMatchParams.vue b/ui/src/views/options/overlays/SceneMatchParams.vue new file mode 100644 index 000000000..5bbdcdb1c --- /dev/null +++ b/ui/src/views/options/overlays/SceneMatchParams.vue @@ -0,0 +1,275 @@ + + + + + diff --git a/ui/src/views/options/sections/InterfaceAdvanced.vue b/ui/src/views/options/sections/InterfaceAdvanced.vue index b466a3eb5..4822c5144 100644 --- a/ui/src/views/options/sections/InterfaceAdvanced.vue +++ b/ui/src/views/options/sections/InterfaceAdvanced.vue @@ -7,7 +7,8 @@ - + + @@ -88,6 +89,18 @@ + + +
+
+ +
+
+ + Save @@ -96,6 +109,62 @@ + +
+
+
+ + + + Link Scenes after Scene Scraping + + + + + + + Include Scenes from Alternate Sites in File Matching + + + + + + + Include Scenes from Alternate Sites when filtering/sorting Scenes for Scripts + + + + + + + + + + + + + + Clear scene links - keep edits + Clear scene links + Re-link scenes + + + Save + +
+
+
+ @@ -115,10 +184,11 @@ export default { scraperCompany: '', scraperAvatar: '', scraperFieldsValid: false, + masterSiteId: '', } }, methods: { - save () { + save () { this.$store.dispatch('optionsAdvanced/save') }, validateScraperFields() { @@ -137,7 +207,8 @@ export default { scraperUrl: this.scraperUrl, scraperName: this.scraperName, scraperCompany: this.scraperCompany, - scraperAvatar: this.scraperAvatar + scraperAvatar: this.scraperAvatar, + masterSiteId: this.masterSiteId, } }) @@ -148,6 +219,15 @@ export default { scrapeXbvrActors() { ky.get('/api/extref/generic/scrape_all') }, + clearAltSrcKeepEdits () { + ky.delete(`/api/extref/delete_extref_source_links/keep_manual`, { json: {external_source: 'alternate scene %'} }); + }, + clearAltSrc () { + ky.delete(`/api/extref/delete_extref_source_links/all`, { json: {external_source: 'alternate scene %'} }); + }, + relinkAltSrc () { + ky.get('/api/task/relink_alt_aource_scenes') + }, }, computed: { showInternalSceneId: { @@ -180,7 +260,6 @@ export default { }, set (value) { this.$store.state.optionsAdvanced.advanced.stashApiKey = value - } }, useImperialEntry: { @@ -201,6 +280,62 @@ export default { } }, + useAltSrcInFileMatching: { + get () { + return this.$store.state.optionsAdvanced.advanced.useAltSrcInFileMatching + }, + set (value) { + this.$store.state.optionsAdvanced.advanced.useAltSrcInFileMatching = value + } + }, + linkScenesAfterSceneScraping: { + get () { + return this.$store.state.optionsAdvanced.advanced.linkScenesAfterSceneScraping + }, + set (value) { + this.$store.state.optionsAdvanced.advanced.linkScenesAfterSceneScraping = value + } + + }, + useAltSrcInScriptFilters: { + get () { + return this.$store.state.optionsAdvanced.advanced.useAltSrcInScriptFilters + }, + set (value) { + this.$store.state.optionsAdvanced.advanced.useAltSrcInScriptFilters = value + } + }, + ignoreReleasedBefore: { + get () { + return new Date(this.$store.state.optionsAdvanced.advanced.ignoreReleasedBefore) + }, + set (value) { + this.$store.state.optionsAdvanced.advanced.ignoreReleasedBefore = value + } + }, + listOfMainSites: { + get () { + const items = this.$store.state.optionsSites.items + let sites = [] + for (let i=0; i < items.length; i++) { + if (items[i].master_site_id == '') { + sites.push(items[i].name) + } + } + return sites + }, + set (value) { + if (value == "") { + this.masterSiteId="" + } else { + const siteFound = this.$store.state.optionsSites.items.find(site => site.master_site_id == '' && site.name==value); + this.masterSiteId=siteFound.id + } + } + }, + showMatchParamsOverlay () { + return this.$store.state.overlay.sceneMatchParams.show + }, isLoading: function () { return this.$store.state.optionsAdvanced.loading } diff --git a/ui/src/views/options/sections/OptionsSceneCreate.vue b/ui/src/views/options/sections/OptionsSceneCreate.vue index 78331b96a..81806bc09 100644 --- a/ui/src/views/options/sections/OptionsSceneCreate.vue +++ b/ui/src/views/options/sections/OptionsSceneCreate.vue @@ -105,6 +105,12 @@ export default { }, mounted () { this.$store.dispatch('optionsVendor/load') + + if (this.$store.state.optionsSceneCreate.scrapeScene!='') { + this.scrapeUrl=this.$store.state.optionsSceneCreate.scrapeScene + this.$store.commit('optionsSceneCreate/setScrapeScene', "") + this.scrapeSingleScene() + } }, methods: { addScene(showEdit) { diff --git a/ui/src/views/options/sections/OptionsSceneDataImportExport.vue b/ui/src/views/options/sections/OptionsSceneDataImportExport.vue index 687a5f710..df6836d24 100644 --- a/ui/src/views/options/sections/OptionsSceneDataImportExport.vue +++ b/ui/src/views/options/sections/OptionsSceneDataImportExport.vue @@ -111,13 +111,6 @@ Include Actor Edits - - - Include External References - - Toggle Includes

{{ isImport ? "Import Settings" : "Export Settings"}}

@@ -150,6 +143,30 @@ Include Scraper Settings +
+
+ + + Include External References + + +
+
+ +
+
+ +
+
+ +
+
{ + ky.get('/api/task/bundle/backup', { timeout: false, searchParams: { allSites: this.allSites == "true", onlyIncludeOfficalSites: this.onlyIncludeOfficalSites, inclScenes: this.includeScenes, inclHistory: this.includeHistory, + inclLinks: this.includeFileLinks, inclCuepoints: this.includeCuepoints, inclActions: this.includeActions, inclPlaylists: this.includePlaylists, inclActorAkas: this.includeActorAkas, inclTagGroups: this.includeTagGroups, + inclVolumes: this.includeVolumes, inclExtRefs: this.includeExternalReferences, inclSites: this.includeSites, inclActors: this.includeActors,inclActorActions: this.inclActorActions, + inclConfig: this.includeConfig, extRefSubset: this.extRefSubset, playlistId: this.currentPlaylist, download: true } }).json().then(data => { const link = document.createElement('a') link.href = this.myUrl link.click() @@ -331,13 +355,13 @@ export default { this.includeActors = !this.includeActors this.includeActorAkas = !this.includeActorAkas this.inclActorActions = !this.inclActorActions - this.includeExternalReferences = !this.includeExternalReferences }, toggleSettingsIncludes () { this.includeTagGroups = !this.includeTagGroups this.includePlaylists = !this.includePlaylists this.includeVolumes=!this.includeVolumes this.includeSites=!this.includeSites + this.includeExternalReferences = !this.includeExternalReferences this.includeConfig=!this.includeConfig }, } diff --git a/ui/src/views/options/sections/OptionsSceneDataScrapers.vue b/ui/src/views/options/sections/OptionsSceneDataScrapers.vue index a8c4e4a3f..a3989d29f 100644 --- a/ui/src/views/options/sections/OptionsSceneDataScrapers.vue +++ b/ui/src/views/options/sections/OptionsSceneDataScrapers.vue @@ -44,7 +44,7 @@ - +
@@ -58,18 +58,32 @@ {{$t('Scrape Single Scene')}} - + {{$t('Force update scenes')}} - + + {{$t('Remove Scene Links')}} + + + {{$t('Remove Scene Links (Keep edits)')}} + + {{$t('Delete scraped scenes')}} - + {{$t('Scrape Actor Details from Site')}}
+ + + + + + {{getMasterSiteName(props.row.master_site_id)}} + +
@@ -241,16 +255,43 @@ export default { }) this.$buefy.toast.open(`Scenes from ${site} will be updated on next scrape`) }, - deleteScenes (site, scraper) { + deleteScenes (site) { this.$buefy.dialog.confirm({ title: this.$t('Delete scraped scenes'), - message: `You're about to delete scraped scenes for ${site}. Previously matched files will return to unmatched state.`, + message: `You're about to delete scraped scenes for ${site.name}.`, type: 'is-danger', hasIcon: true, onConfirm: function () { - ky.post('/api/options/scraper/delete-scenes', { - json: { scraper_id: scraper } - }) + if (site.master_site_id==""){ + ky.post('/api/options/scraper/delete-scenes', { + json: { scraper_id: site.id } + }) + } else { + const external_source = 'alternate scene ' + site.id + ky.delete(`/api/extref/delete_extref_source`, { + json: {external_source: external_source} + }); + } + } + }) + }, + removeSceneLinks (site, all) { + this.$buefy.dialog.confirm({ + title: this.$t('Remove Scene Links'), + message: `You're about to remove links for scenes from ${site.name}. Scenes will be relinked after the next scrape.`, + type: 'is-warning', + hasIcon: true, + onConfirm: function () { + const external_source = 'alternate scene ' + site.id + if (all) { + ky.delete(`/api/extref/delete_extref_source_links/all`, { + json: {external_source: external_source} + }); + } else { + ky.delete(`/api/extref/delete_extref_source_links/keep_manual`, { + json: {external_source: external_source} + }); + } } }) }, @@ -267,6 +308,15 @@ export default { } this.isLoading=false }, + editMatchParams(site){ + this.$store.commit('overlay/showSceneMatchParams', { site: site }) + }, + getMasterSiteName(siteId){ + if (siteId=="") { + return "" + } + return this.scraperList.find(element => element.id === siteId).name; + }, parseISO, formatDistanceToNow }, diff --git a/ui/src/views/options/sections/Schedules.vue b/ui/src/views/options/sections/Schedules.vue index 34f7dbc09..18df4b3c4 100644 --- a/ui/src/views/options/sections/Schedules.vue +++ b/ui/src/views/options/sections/Schedules.vue @@ -10,6 +10,7 @@ +
@@ -209,6 +210,46 @@
{{ delayStartMsg(stashdbRescrapeStartDelay) }}
+
+ + Enable schedule + + + +
{{`Run every ${this.linkScenesHourInterval} hour${this.linkScenesHourInterval > 1 ? 's': ''}`}}
+ + + Limit time of day + +
+ + + 00:00 + 06:00 + 12:00 + 18:00 + Midnight + 06:00 + 12:00 + 18:00 + 00:00 + +
{{`${this.timeRange[this.linkScenesTimeRange[0]]} - ${this.timeRange[this.linkScenesTimeRange[1]]}`}}
+ + + +
{{ minutesStartMsg(linkScenesMinuteStart) }}
+ +

+ Linking Scenes will not start after the Time Window Ends +

+
+
+ + +
{{ delayStartMsg(linkScenesStartDelay) }}
+ +
Save settings @@ -271,6 +312,13 @@ export default { lastStashdbRescrapeTimeRange: [0,23], useStashdbRescrapeTimeRange: false, stashdbRescrapeStartDelay: 0, + linkScenesEnabled: false, + linkScenesTimeRange:[0,23], + linkScenesHourInterval: 0, + linkScenesMinuteStart: 0, + lastlinkScenesTimeRange: [0,23], + useLinkScenesTimeRange: false, + linkScenesStartDelay: 0, timeRange: ['00:00', '01:00', '02:00', '03:00', '04:00', '05:00', '06:00', '07:00', '08:00', '09:00', '10:00', '11:00', '12:00', '13:00', '14:00', '15:00', '16:00', '17:00', '18:00', '19:00', '20:00', '21:00', '22:00', '23:00', '00:00', '01:00', '02:00', '03:00', '04:00', '05:00', '06:00', '07:00', '08:00', '09:00', '10:00', '11:00', @@ -312,6 +360,10 @@ export default { this.stashdbRescrapeTimeRange = this.restrictTo24Hours(this.stashdbRescrapeTimeRange, this.lastStashdbRescrapeTimeRange) this.lastStashdbRescrapeTimeRange = this.stashdbRescrapeTimeRange }, + restrictLinkScenesTo24Hours () { + this.linkScenesTimeRange = this.restrictTo24Hours(this.linkScenesTimeRange, this.lastLinkScenesTimeRange) + this.lastLinkScenesTimeRange = this.LinkScenesTimeRange + }, restrictTo24Hours (timeRange, lastTimeRange) { // check the first time is not in the second 24 hours, no need, should be in the first 24 hours if (timeRange[0] > 23) { @@ -353,6 +405,10 @@ export default { this.stashdbRescrapeHourInterval = data.config.cron.stashdbRescrapeSchedule.hourInterval this.useStashdbRescrapeTimeRange = data.config.cron.stashdbRescrapeSchedule.useRange this.stashdbRescrapeMinuteStart = data.config.cron.stashdbRescrapeSchedule.minuteStart + this.linkScenesEnabled = data.config.cron.linkScenesSchedule.enabled + this.linkScenesHourInterval = data.config.cron.linkScenesSchedule.hourInterval + this.useLinkScenesTimeRange = data.config.cron.linkScenesSchedule.useRange + this.linkScenesMinuteStart = data.config.cron.linkScenesSchedule.minuteStart if (data.config.cron.rescrapeSchedule.hourStart > data.config.cron.rescrapeSchedule.hourEnd) { this.rescrapeTimeRange = [data.config.cron.rescrapeSchedule.hourStart, data.config.cron.rescrapeSchedule.hourEnd + 24] } else { @@ -379,12 +435,19 @@ export default { } else { this.stashdbRescrapeTimeRange = [data.config.cron.stashdbRescrapeSchedule.hourStart, data.config.cron.stashdbRescrapeSchedule.hourEnd] } + + if (data.config.cron.linkScenesSchedule.hourStart > data.config.cron.linkScenesSchedule.hourEnd) { + this.linkScenesTimeRange = [data.config.cron.linkScenesSchedule.hourStart, data.config.cron.linkScenesSchedule.hourEnd + 24] + } else { + this.linkScenesTimeRange = [data.config.cron.linkScenesSchedule.hourStart, data.config.cron.linkScenesSchedule.hourEnd] + } this.rescrapeStartDelay = data.config.cron.rescrapeSchedule.runAtStartDelay this.rescanStartDelay = data.config.cron.rescanSchedule.runAtStartDelay this.previewStartDelay = data.config.cron.previewSchedule.runAtStartDelay this.actorRescrapeStartDelay = data.config.cron.actorRescrapeSchedule.runAtStartDelay this.stashdbRescrapeStartDelay = data.config.cron.stashdbRescrapeSchedule.runAtStartDelay + this.linkScenesStartDelay = data.config.cron.linkScenesSchedule.runAtStartDelay this.isLoading = false }) }, @@ -446,7 +509,14 @@ export default { stashdbRescrapeMinuteStart: this.stashdbRescrapeMinuteStart, stashdbRescrapeHourStart: this.stashdbRescrapeTimeRange[0], stashdbRescrapeHourEnd: this.stashdbRescrapeTimeRange[1], - stashdbRescrapeStartDelay:this.stashdbRescrapeStartDelay + stashdbRescrapeStartDelay:this.stashdbRescrapeStartDelay, + linkScenesEnabled: this.linkScenesEnabled, + linkScenesHourInterval: this.linkScenesHourInterval, + linkScenesUseRange: this.useLinkScenesTimeRange, + linkScenesMinuteStart: this.linkScenesMinuteStart, + linkScenesHourStart: this.linkScenesTimeRange[0], + linkScenesHourEnd: this.linkScenesTimeRange[1], + linkScenesStartDelay:this.linkScenesStartDelay } }) .json() diff --git a/ui/src/views/scenes/Details.vue b/ui/src/views/scenes/Details.vue index 89762c466..6b49693d8 100644 --- a/ui/src/views/scenes/Details.vue +++ b/ui/src/views/scenes/Details.vue @@ -43,7 +43,7 @@ - +
-
+
@@ -192,7 +215,7 @@
- +
@@ -241,7 +264,7 @@
- +
@@ -310,7 +333,7 @@
- +
{{ historySessionsCount }} view sessions, total duration @@ -332,7 +355,7 @@
- +
@@ -357,9 +380,9 @@
- -
@@ -411,6 +434,8 @@ export default { sortMultiple: true, castimages: [], searchfields: [], + alternateSources: [], + waitingForQuickFind: false, } }, computed: { @@ -443,6 +468,7 @@ export default { this.castimages = imgs.filter((img) => { return img.src !== ''; }); + this.getSearchFields(item.id) return item }, // Properties for gallery @@ -523,6 +549,34 @@ export default { .indexOf(this.cuepointName.toString().toLowerCase()) >= 0 }) }, + displayingAlternateSource () { + // displayingAlternateSource indicates we aren't displaying a real xbvr scene from the scenes table, + // so functions like watchlist, ratings, etc don't apply + // we are displaying scene data serialized and saved in the external_references table + if ( this.$store.state.overlay.details.altsrc != null) return true + return false + }, + async getAlternateSceneSources() { + this.alternateSources = []; + if (this.displayingAlternateSource) return 0 + try { + const response = await ky.get('/api/scene/alternate_source/' + this.item.id).json(); + if (response==null){ + return 0 + } + response.forEach(altsrc => { + if (altsrc.external_source.startsWith("alternate scene ")) { + this.alternateSources.push(altsrc) + } + }); + return this.alternateSources.length; + } catch (error) { + return 0; // Return 0 or handle error as needed + } + }, + quickFindOverlayState() { + return this.$store.state.overlay.quickFind.show + } }, mounted () { this.setupPlayer() @@ -533,8 +587,29 @@ export default { this.cuepointPositionTags = data.positions this.cuepointActTags.unshift("") this.cuepointPositionTags.unshift("") - }) - this.getSearchFields() + }) +}, +watch:{ + quickFindOverlayState(newVal, oldVal){ + if (newVal == true) { + return + } + if (this.waitingForQuickFind){ + this.waitingForQuickFind = false + if (this.$store.state.overlay.quickFind.selectedScene != null && this.$store.state.overlay.quickFind.selectedScene.id > 0) { + this.$buefy.dialog.confirm({ + title: 'Relink scene', + message: `Do you wish to link this scene to ${this.$store.state.overlay.quickFind.selectedScene.title}`, + type: 'is-info is-wide', + hasIcon: true, + id: 'heh', + onConfirm: () => { + this.handleRelinkExtRef() + } + }) + } + } + }, }, methods: { setupPlayer () { @@ -556,7 +631,7 @@ export default { return event.which === 27 }, handler: (player, options, event) => { - this.player.dispose() + if (!this.displayingAlternateSource) this.player.dispose() this.$store.commit('overlay/hideDetails') } }, @@ -786,7 +861,7 @@ export default { }) }, close () { - this.player.dispose() + if (!this.displayingAlternateSource) this.player.dispose() this.$store.commit('overlay/hideDetails') }, humanizeSeconds (seconds) { @@ -805,23 +880,21 @@ export default { }, nextScene () { const data = this.$store.getters['sceneList/nextScene'](this.item) - if (data !== null) { + if (data !== null && !this.displayingAlternateSource) { this.$store.commit('overlay/showDetails', { scene: data }) this.activeMedia = 0 this.carouselSlide = 0 this.updatePlayer(undefined, '180') } - this.getSearchFields() }, prevScene () { const data = this.$store.getters['sceneList/prevScene'](this.item) - if (data !== null) { + if (data !== null && !this.displayingAlternateSource) { this.$store.commit('overlay/showDetails', { scene: data }) this.activeMedia = 0 this.carouselSlide = 0 this.updatePlayer(undefined, '180') } - this.getSearchFields() }, playerStepBack (interval) { const wasPlaying = !this.player.paused() @@ -939,18 +1012,122 @@ export default { hideTooltip(idx) { this.castimages[idx].visible = false; }, - getSearchFields() { + getSearchFields(id) { // load search fields - if (this.$store.state.optionsAdvanced.advanced.showSceneSearchField) { + this.searchfields = [] + if (this.$store.state.optionsAdvanced.advanced.showSceneSearchField && !this.displayingAlternateSource) { ky.get('/api/scene/searchfields', { searchParams: { - q: this.item.id + q: id }, }).json().then(data => { this.searchfields = data }) } }, + showExtRefScene (altsrc) { + const extdata = JSON.parse(altsrc.external_data); + if (extdata.scene.cast == null) + { + extdata.scene.cast = [] + } + this.$store.commit('overlay/showDetails', { scene: extdata.scene, altsrc: altsrc, prevscene: this.item, query_for_altsrc: extdata.query }) + this.activeTab = 0 + }, + searchAlternateSourceScene() { + // search for a new scene to link to the alternate source scene + const q = this.$store.state.overlay.details.query_for_altsrc == "" ? this.item.title : this.$store.state.overlay.details.query_for_altsrc + this.$store.commit('overlay/showQuickFind', { searchString: q, displaySelectedScene: false }) + this.waitingForQuickFind = true + }, + async handleRelinkExtRef() { + const response = await ky.post(`/api/extref/edit_link`, { + json: { + external_source: this.$store.state.overlay.details.altsrc.external_source, + external_id: this.$store.state.overlay.details.altsrc.external_id, + internal_table: "scenes", + internal_db_id: this.$store.state.overlay.quickFind.selectedScene.id, + internal_name_id: this.$store.state.overlay.quickFind.selectedScene.scene_id, + match_type: 99999 + } + }); + if (response.status === 200) { + this.$store.state.overlay.details.prevscene = this.$store.state.overlay.quickFind.selectedScene; + this.$buefy.toast.open({ message: `The scene was sucessfully relinked to a new Scene`, type: 'is-primary', duration: 3000 }); + } + }, + async scrapeScene() { + this.$buefy.dialog.confirm({ + title: 'Scrape & Create Scene', + message: `Do you wish to create a seperate XBVR scene from this linked scene ${this.$store.state.overlay.details.altsrc.url}`, + type: 'is-info is-wide', + hasIcon: true, + id: 'heh', + onConfirm: () => { + const url = this.$store.state.overlay.details.altsrc.url + this.$store.state.overlay.details.altsrc = null + this.$store.commit('overlay/hideDetails') + // call the options screen passing the url in state + this.$store.commit('optionsSceneCreate/setScrapeScene', url ) + this.$store.commit('optionsSceneCreate/showSceneCreate', true ) + this.$router.push({ path: '/options'}) + } + }) + + }, + async refreshExtRef() { + this.$buefy.dialog.confirm({ + title: 'Continue?', + message: `This will remove the scene, rescrape the site to relink it to an XBVR scene`, + type: 'is-info is-wide', + hasIcon: true, + id: 'heh', + onConfirm: () => { + this.handleRefreshExtRef() + } + }) + }, + async handleRefreshExtRef() { + const response = await ky.delete(`/api/extref/delete_extref`, { + json: { + external_source: this.$store.state.overlay.details.altsrc.external_source, + external_id: this.$store.state.overlay.details.altsrc.external_id, + } + }); + if (response.status === 200) { + this.$store.state.overlay.details.prevscene = this.$store.state.overlay.quickFind.selectedScene; + this.$buefy.toast.open({ message: `The scene was removed, ready to rescan`, type: 'is-primary', duration: 3000 }); + } + }, + flagExtRefDeleted() { + let confirmed = false + this.$buefy.dialog.confirm({ + title: 'Continue?', + message: `This will unlink the scene and prevent it from relinking to any scene. This cannot be undone`, + type: 'is-danger is-wide', + hasIcon: true, + id: 'heh', + onConfirm: () => { + this.handleFlagExtRefDeleted() + }, + }) + }, + async handleFlagExtRefDeleted() { + const response = await ky.post(`/api/extref/edit_link`, { + json: { + external_source: this.$store.state.overlay.details.altsrc.external_source, + external_id: this.$store.state.overlay.details.altsrc.external_id, + internal_table: "scenes", + internal_db_id: 0, + internal_name_id: "deleted", + match_type: -1 + } + }); + if (response.status === 200) { + this.$store.state.overlay.details.prevscene = this.$store.state.overlay.quickFind.selectedScene; + this.$buefy.toast.open({ message: `The scene was unlinked and will not be relinked to any scene`, type: 'is-primary', duration: 3000 }); + } + }, format, parseISO, prettyBytes, @@ -1112,4 +1289,8 @@ span.is-active img { .tooltip img { max-width: 100%; max-height: 100%; +} +.altsrc-image-wrapper { + display: inline-block; + margin-left: 5px; } diff --git a/ui/src/views/scenes/EditScene.vue b/ui/src/views/scenes/EditScene.vue index a6b7cb69a..8134da00a 100644 --- a/ui/src/views/scenes/EditScene.vue +++ b/ui/src/views/scenes/EditScene.vue @@ -9,7 +9,7 @@
-

{{ $t('Edit scene details') }}

+

{{ this.scene.id == 0 ? $t('Display scene details') : $t('Edit scene details') }}

@@ -97,7 +97,7 @@
- + {{ $t('Save Scene Details') }} {{ $t('Delete Scene') }} diff --git a/ui/src/views/scenes/Filters.vue b/ui/src/views/scenes/Filters.vue index a6738904b..3fcd268bd 100644 --- a/ui/src/views/scenes/Filters.vue +++ b/ui/src/views/scenes/Filters.vue @@ -57,6 +57,7 @@ +
diff --git a/ui/src/views/scenes/SceneCard.vue b/ui/src/views/scenes/SceneCard.vue index 2a596fe49..7a8d76256 100644 --- a/ui/src/views/scenes/SceneCard.vue +++ b/ui/src/views/scenes/SceneCard.vue @@ -66,8 +66,18 @@ {{item.site}}
{{format(parseISO(item.release_date), "yyyy-MM-dd")}} - + +
+
+ + + Image + + + +
+
@@ -82,16 +92,18 @@ import EditButton from '../../components/EditButton' import TrailerlistButton from '../../components/TrailerlistButton' import HiddenButton from '../../components/HiddenButton' import ky from 'ky' +import VueLoadImage from 'vue-load-image' export default { name: 'SceneCard', props: { item: Object, reRead: Boolean }, - components: { WatchlistButton, FavouriteButton, WishlistButton, WatchedButton, EditButton, TrailerlistButton, HiddenButton }, + components: { WatchlistButton, FavouriteButton, WishlistButton, WatchedButton, EditButton, TrailerlistButton, HiddenButton, VueLoadImage }, data () { return { preview: false, format, - parseISO + parseISO, + alternateSources: [], } }, computed: { @@ -141,6 +153,23 @@ export default { } return this.$store.state.optionsWeb.web.isAvailOpacity / 100 }, + async getAlternateSceneSources() { + try { + const response = await ky.get('/api/scene/alternate_source/' + this.item.id).json(); + this.alternateSources = []; + if (response==null){ + return 0 + } + response.forEach(altsrc => { + if (altsrc.external_source.startsWith("alternate scene ") || altsrc.external_source == "stashdb scene") { + this.alternateSources.push(altsrc) + } + }); + return this.alternateSources.length; + } catch (error) { + return 0; // Return 0 or handle error as needed + } + }, }, methods: { getImageURL (u) { @@ -267,4 +296,10 @@ export default { border-radius: 0.25rem; } +.altsrc-image-wrapper { + display: inline-block; + margin-right: 5px; + margin-top: 3px; +} + From fad56ab27a5e07ca98e74ba8f9c7a9957fd23035 Mon Sep 17 00:00:00 2001 From: toshski <104477758+toshski@users.noreply.github.com> Date: Thu, 25 Jan 2024 04:46:42 +1300 Subject: [PATCH 10/11] scraper: Add MembersUrl link for Tmwvrnet Scenes (#1595) * Set up members url for tmwvrnet * Make member domain configurable * go fmt * go fmt --------- Co-authored-by: crwxaj <52156245+crwxaj@users.noreply.github.com> --- pkg/config/config.go | 5 +++++ pkg/migrations/migrations.go | 12 ++++++++++++ pkg/scrape/tmwvrnet.go | 3 +++ 3 files changed, 20 insertions(+) diff --git a/pkg/config/config.go b/pkg/config/config.go index 257024a4f..3b8bfa2ea 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -170,6 +170,11 @@ type ObjectConfig struct { Storage struct { MatchOhash bool `default:"false" json:"match_ohash"` } `json:"storage"` + ScraperSettings struct { + TMWVRNet struct { + TmwMembersDomain string `default:"members.tmwvrnet.com" json:"tmwMembersDomain"` + } `json:"tmwvrnet"` + } `json:"scraper_settings"` } var ( diff --git a/pkg/migrations/migrations.go b/pkg/migrations/migrations.go index 035fbed79..e4cb59b69 100644 --- a/pkg/migrations/migrations.go +++ b/pkg/migrations/migrations.go @@ -1911,6 +1911,18 @@ func Migrate() { return nil }, }, + { + ID: "0075-Update-tmwvrnet-members", + Migrate: func(tx *gorm.DB) error { + sql := `update scenes set member_url = replace(replace(scene_url, 'https://tmwvrnet.com/trailers/', 'https://members.tmwvrnet.com/scenes/'), '.html', '_vids.html') where scene_url like 'https://tmwvrnet.com/trailers/%';` + err := tx.Exec(sql).Error + if err == nil { + + err = tx.Exec(sql).Error + } + return err + }, + }, }) if err := m.Migrate(); err != nil { diff --git a/pkg/scrape/tmwvrnet.go b/pkg/scrape/tmwvrnet.go index c5a083400..17f826d59 100644 --- a/pkg/scrape/tmwvrnet.go +++ b/pkg/scrape/tmwvrnet.go @@ -10,6 +10,7 @@ import ( "github.com/mozillazg/go-slugify" "github.com/nleeper/goment" "github.com/thoas/go-funk" + "github.com/xbapps/xbvr/pkg/config" "github.com/xbapps/xbvr/pkg/models" ) @@ -30,6 +31,8 @@ func TmwVRnet(wg *sync.WaitGroup, updateSite bool, knownScenes []string, out cha sc.Studio = "TeenMegaWorld" sc.Site = siteID sc.HomepageURL = strings.Split(e.Request.URL.String(), "?")[0] + sc.MembersUrl = strings.Replace(sc.HomepageURL, "https://tmwvrnet.com/trailers/", "https://"+config.Config.ScraperSettings.TMWVRNet.TmwMembersDomain+"/scenes/", 1) + sc.MembersUrl = strings.Replace(sc.MembersUrl, ".html", "_vids.html", 1) // Date & Duration e.ForEach(`.video-info-data`, func(id int, e *colly.HTMLElement) { From 20ed68aeca9183d749f0e9d84e177253ba29505e Mon Sep 17 00:00:00 2001 From: crwxaj <52156245+crwxaj@users.noreply.github.com> Date: Wed, 24 Jan 2024 17:30:00 +0100 Subject: [PATCH 11/11] minor: UI fixes (#1606) Co-authored-by: crwxaj --- go.mod | 1 - go.sum | 53 +------------------ pkg/models/model_external_reference.go | 2 +- ui/src/locales/en-GB.json | 15 +++--- .../options/overlays/SceneMatchParams.vue | 6 +-- .../options/sections/InterfaceAdvanced.vue | 4 +- .../sections/OptionsSceneDataScrapers.vue | 14 +++++ 7 files changed, 30 insertions(+), 65 deletions(-) diff --git a/go.mod b/go.mod index 4242377e2..a030f264e 100644 --- a/go.mod +++ b/go.mod @@ -149,6 +149,5 @@ require ( google.golang.org/appengine v1.6.7 // indirect google.golang.org/protobuf v1.31.0 // indirect gopkg.in/sourcemap.v1 v1.0.5 // indirect - gopkg.in/yaml.v2 v2.4.0 // indirect willnorris.com/go/gifresize v1.0.0 // indirect ) diff --git a/go.sum b/go.sum index 17ab3b308..7237f4f1e 100644 --- a/go.sum +++ b/go.sum @@ -42,10 +42,6 @@ github.com/bits-and-blooms/bitset v1.2.0 h1:Kn4yilvwNtMACtf1eYDlG8H77R07mZSPbMjL github.com/bits-and-blooms/bitset v1.2.0/go.mod h1:gIdJ4wp64HaoK2YrL1Q5/N7Y16edYb8uY+O0FJTyyDA= github.com/blevesearch/bleve/v2 v2.3.10 h1:z8V0wwGoL4rp7nG/O3qVVLYxUqCbEwskMt4iRJsPLgg= github.com/blevesearch/bleve/v2 v2.3.10/go.mod h1:RJzeoeHC+vNHsoLR54+crS1HmOWpnH87fL70HAUCzIA= -github.com/blevesearch/bleve_index_api v1.1.3 h1:aNyMEiWFviY/1zYm7JCr2lZRIiYX0TMtz3oymxxbApc= -github.com/blevesearch/bleve_index_api v1.1.3/go.mod h1:PbcwjIcRmjhGbkS/lJCpfgVSMROV6TRubGGAODaK1W8= -github.com/blevesearch/bleve_index_api v1.1.4 h1:n9Ilxlb80g9DAhchR95IcVrzohamDSri0wPnkKnva50= -github.com/blevesearch/bleve_index_api v1.1.4/go.mod h1:PbcwjIcRmjhGbkS/lJCpfgVSMROV6TRubGGAODaK1W8= github.com/blevesearch/bleve_index_api v1.1.5 h1:0q05mzu6GT/kebzqKywCpou/eUea9wTKa7kfqX7QX+k= github.com/blevesearch/bleve_index_api v1.1.5/go.mod h1:PbcwjIcRmjhGbkS/lJCpfgVSMROV6TRubGGAODaK1W8= github.com/blevesearch/geo v0.1.18 h1:Np8jycHTZ5scFe7VEPLrDoHnnb9C4j636ue/CGrhtDw= @@ -107,8 +103,6 @@ github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+m github.com/emicklei/go-restful-openapi/v2 v2.9.1 h1:Of8B1rXdG81il5TTiSY+9Qrh7pYOr8aLdynHIpvo7fM= github.com/emicklei/go-restful-openapi/v2 v2.9.1/go.mod h1:VKNgZyYviM1hnyrjD9RDzP2RuE94xTXxV+u6MGN4v4k= github.com/emicklei/go-restful/v3 v3.7.3/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= -github.com/emicklei/go-restful/v3 v3.11.0 h1:rAQeMHw1c7zTmncogyy8VvRZwtkmkZ4FxERmMY4rD+g= -github.com/emicklei/go-restful/v3 v3.11.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= github.com/emicklei/go-restful/v3 v3.11.2 h1:1onLa9DcsMYO9P+CXaL0dStDqQ2EHHXLiz+BtnqkLAU= github.com/emicklei/go-restful/v3 v3.11.2/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= @@ -135,29 +129,19 @@ github.com/getlantern/ops v0.0.0-20190325191751-d70cb0d6f85f/go.mod h1:D5ao98qkA github.com/getlantern/systray v1.2.2 h1:dCEHtfmvkJG7HZ8lS/sLklTH4RKUcIsKrAD9sThoEBE= github.com/getlantern/systray v1.2.2/go.mod h1:pXFOI1wwqwYXEhLPm9ZGjS2u/vVELeIgNMY5HvhHhcE= github.com/go-openapi/jsonpointer v0.19.3/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= -github.com/go-openapi/jsonpointer v0.19.5 h1:gZr+CIYByUqjcgeLXnQu2gHYQC9o73G2XUeOFYEICuY= github.com/go-openapi/jsonpointer v0.19.5/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= github.com/go-openapi/jsonpointer v0.20.2 h1:mQc3nmndL8ZBzStEo3JYF8wzmeWffDH4VbXz58sAx6Q= github.com/go-openapi/jsonpointer v0.20.2/go.mod h1:bHen+N0u1KEO3YlmqOjTT9Adn1RfD91Ar825/PuiRVs= github.com/go-openapi/jsonreference v0.19.6/go.mod h1:diGHMEHg2IqXZGKxqyvWdfWU/aim5Dprw5bqpKkTvns= -github.com/go-openapi/jsonreference v0.20.0 h1:MYlu0sBgChmCfJxxUKZ8g1cPWFOB37YSZqewK7OKeyA= -github.com/go-openapi/jsonreference v0.20.0/go.mod h1:Ag74Ico3lPc+zR+qjn4XBUmXymS4zJbYVCZmcgkasdo= github.com/go-openapi/jsonreference v0.20.4 h1:bKlDxQxQJgwpUSgOENiMPzCTBVuc7vTdXSSgNeAhojU= github.com/go-openapi/jsonreference v0.20.4/go.mod h1:5pZJyJP2MnYCpoeoMAql78cCHauHj0V9Lhc506VOpw4= github.com/go-openapi/spec v0.20.4/go.mod h1:faYFR1CvsJZ0mNsmsphTMSoRrNV3TEDoAM7FOEWeq8I= -github.com/go-openapi/spec v0.20.9 h1:xnlYNQAwKd2VQRRfwTEI0DcK+2cbuvI/0c7jx3gA8/8= -github.com/go-openapi/spec v0.20.9/go.mod h1:2OpW+JddWPrpXSCIX8eOx7lZ5iyuWj3RYR6VaaBKcWA= -github.com/go-openapi/spec v0.20.13 h1:XJDIN+dLH6vqXgafnl5SUIMnzaChQ6QTo0/UPMbkIaE= -github.com/go-openapi/spec v0.20.13/go.mod h1:8EOhTpBoFiask8rrgwbLC3zmJfz4zsCUueRuPM6GNkw= github.com/go-openapi/spec v0.20.14 h1:7CBlRnw+mtjFGlPDRZmAMnq35cRzI91xj03HVyUi/Do= github.com/go-openapi/spec v0.20.14/go.mod h1:8EOhTpBoFiask8rrgwbLC3zmJfz4zsCUueRuPM6GNkw= github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= -github.com/go-openapi/swag v0.19.15 h1:D2NRCBzS9/pEY3gP9Nl8aDqGUcPFrwG2p+CNFrLyrCM= github.com/go-openapi/swag v0.19.15/go.mod h1:QYRuS/SOXUCsnplDa677K7+DxSOj6IPNl/eQntq43wQ= github.com/go-openapi/swag v0.22.6 h1:dnqg1XfHXL9aBxSbktBqFR5CxVyVI+7fYWhAf1JOeTw= github.com/go-openapi/swag v0.22.6/go.mod h1:Gl91UqO+btAM0plGGxHqJcQZ1ZTy6jbmridBTsDy8A0= -github.com/go-resty/resty/v2 v2.10.0 h1:Qla4W/+TMmv0fOeeRqzEpXPLfTUnR5HZ1+lGs+CkiCo= -github.com/go-resty/resty/v2 v2.10.0/go.mod h1:iiP/OpA0CkcL3IGt1O0+/SIItFUbkkyw5BGXiVdTu+A= github.com/go-resty/resty/v2 v2.11.0 h1:i7jMfNOJYMp69lq7qozJP+bjgzfAzeOhuGlyDrqxT/8= github.com/go-resty/resty/v2 v2.11.0/go.mod h1:iiP/OpA0CkcL3IGt1O0+/SIItFUbkkyw5BGXiVdTu+A= github.com/go-sql-driver/mysql v1.4.1/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= @@ -266,7 +250,6 @@ github.com/lxn/walk v0.0.0-20210112085537-c389da54e794/go.mod h1:E23UucZGqpuUANJ github.com/lxn/win v0.0.0-20210218163916-a377121e959e/go.mod h1:KxxjdtRkfNoYDCUP5ryK7XJJNTnpC8atvtmTheChOtk= github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= -github.com/mailru/easyjson v0.7.6 h1:8yTIVnZgCoiM1TgqoeTl+LfU5Jg6/xL3QhGQnimLYnA= github.com/mailru/easyjson v0.7.6/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= @@ -282,8 +265,6 @@ github.com/mattn/go-isatty v0.0.7/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hd github.com/mattn/go-runewidth v0.0.10/go.mod h1:RAqKPSqVFrSLVXbA8x7dzmKdmGzieGRCM46jaSJTDAk= github.com/mattn/go-sqlite3 v1.10.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc= github.com/mattn/go-sqlite3 v1.14.0/go.mod h1:JIl7NbARA7phWnGvh0LKTyg7S9BA+6gx71ShQilpsus= -github.com/mattn/go-sqlite3 v1.14.18 h1:JL0eqdCOq6DJVNPSvArO/bIV9/P7fbGrV00LZHc+5aI= -github.com/mattn/go-sqlite3 v1.14.18/go.mod h1:2eHXhiwb8IkHr+BDWZGa96P6+rkvnG63S2DGjv9HUNg= github.com/mattn/go-sqlite3 v1.14.19 h1:fhGleo2h1p8tVChob4I9HpmVFIAkKGpiukdrgQbWfGI= github.com/mattn/go-sqlite3 v1.14.19/go.mod h1:2eHXhiwb8IkHr+BDWZGa96P6+rkvnG63S2DGjv9HUNg= github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zkfA9PSy5pEvNWRP0ET0TIVo= @@ -341,14 +322,12 @@ github.com/prometheus/procfs v0.11.1/go.mod h1:eesXgaPo1q7lBpVMoMy0ZOFTth9hBn4W/ github.com/putdotio/go-putio v1.7.1 h1:316PpOMO2a7H73foRxlpHmekeLso07et26Z00YlwQ2A= github.com/putdotio/go-putio v1.7.1/go.mod h1:QhjpLhn3La/ea4FeJlp1qsiaFZDC0EIO8VUe8VEKMV0= github.com/rivo/uniseg v0.1.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= -github.com/robertkrimen/otto v0.2.1 h1:FVP0PJ0AHIjC+N4pKCG9yCDz6LHNPCwi/GKID5pGGF0= -github.com/robertkrimen/otto v0.2.1/go.mod h1:UPwtJ1Xu7JrLcZjNWN8orJaM5n5YEtqL//farB5FlRY= github.com/robertkrimen/otto v0.3.0 h1:5RI+8860NSxvXywDY9ddF5HcPw0puRsd8EgbXV0oqRE= github.com/robertkrimen/otto v0.3.0/go.mod h1:uW9yN1CYflmUQYvAMS0m+ZiNo3dMzRUDQJX0jWbzgxw= github.com/robfig/cron/v3 v3.0.1 h1:WdRxkvbJztn8LMz/QEvLN5sBU+xKpSqwwUO1Pjr4qDs= github.com/robfig/cron/v3 v3.0.1/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzGIFLtro= -github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= -github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog= +github.com/rogpeppe/go-internal v1.11.0 h1:cWPaGQEPrBb5/AsnsZesgZZ9yb1OQ+GOISoDNXVBh4M= +github.com/rogpeppe/go-internal v1.11.0/go.mod h1:ddIwULY96R17DhadqLgMfk9H9tvdUzkipdSkR5nkCZA= github.com/rs/cors v1.10.1 h1:L0uuZVXIKlI1SShY2nhFfo44TYvDPQ1w4oFkUJNfhyo= github.com/rs/cors v1.10.1/go.mod h1:XyqrcTp5zjWr1wsJ8PIRZssZ8b/WMcMf71DJnit4EMU= github.com/rwcarlsen/goexif v0.0.0-20190401172101-9e8deecbddbd h1:CmH9+J6ZSsIjUK3dcGsnCnO41eRBOnY12zwkn5qVwgc= @@ -391,12 +370,6 @@ github.com/x-cray/logrus-prefixed-formatter v0.5.2 h1:00txxvfBM9muc0jiLIEAkAcIMJ github.com/x-cray/logrus-prefixed-formatter v0.5.2/go.mod h1:2duySbKsL6M18s5GU7VPsoEPHyzalCE06qoARUCeBBE= github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8 h1:nIPpBwaJSVYIxUFsDv3M8ofmx9yWTog9BfvIu0q41lo= github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8/go.mod h1:HUYIGzjTL3rfEspMxjDjgmT5uz5wzYJKVo23qUhYTos= -github.com/xo/dburl v0.18.2 h1:9xqcVf+JEV7bcUa1OjCsoax06roohYFdye6xkvBKo50= -github.com/xo/dburl v0.18.2/go.mod h1:B7/G9FGungw6ighV8xJNwWYQPMfn3gsi2sn5SE8Bzco= -github.com/xo/dburl v0.18.3 h1:z271VmL/pk00rAF+0JrwrsOLyQcEBqqjyH3qX7eeJIE= -github.com/xo/dburl v0.18.3/go.mod h1:B7/G9FGungw6ighV8xJNwWYQPMfn3gsi2sn5SE8Bzco= -github.com/xo/dburl v0.20.2 h1:59zqIzahtfQ/X9E6fyp1ziHwjYEFy65opjpyQPmZC7w= -github.com/xo/dburl v0.20.2/go.mod h1:B7/G9FGungw6ighV8xJNwWYQPMfn3gsi2sn5SE8Bzco= github.com/xo/dburl v0.21.1 h1:n5mfH1fh51RQbvuaKKykGslodt8pZqyZJMNohVo2zK0= github.com/xo/dburl v0.21.1/go.mod h1:B7/G9FGungw6ighV8xJNwWYQPMfn3gsi2sn5SE8Bzco= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= @@ -412,12 +385,6 @@ golang.org/x/crypto v0.0.0-20191205180655-e7c4368fe9dd/go.mod h1:LzIPMQfyMNhhGPh golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4= -golang.org/x/crypto v0.15.0 h1:frVn1TEaCEaZcn3Tmd7Y2b5KKPaZ+I32Q2OA3kYp5TA= -golang.org/x/crypto v0.15.0/go.mod h1:4ChreQoLWfG3xLDer1WdlH5NdlQ3+mwnQq1YTKY+72g= -golang.org/x/crypto v0.16.0 h1:mMMrFzRSCF0GvB7Ne27XVtVAaXLrPmgPC7/v0tkwHaY= -golang.org/x/crypto v0.16.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq4= -golang.org/x/crypto v0.17.0 h1:r8bRNjWL3GshPW3gkd+RpvzWrZAwPS49OmTGZ/uhM4k= -golang.org/x/crypto v0.17.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq4= golang.org/x/crypto v0.18.0 h1:PGVlW0xEltQnzFZ55hkuX5+KLyrMYhHld1YHO4AKcdc= golang.org/x/crypto v0.18.0/go.mod h1:R0j02AL6hcrfOiy9T4ZYp/rcWeMxM3L6QYxlOuEG1mg= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= @@ -452,18 +419,10 @@ golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE= -golang.org/x/net v0.18.0 h1:mIYleuAkSbHh0tCv7RvjL3F6ZVbLjq4+R7zbOn3Kokg= -golang.org/x/net v0.18.0/go.mod h1:/czyP5RqHAH4odGYxBJ1qz0+CE5WZ+2j1YgoEo8F2jQ= -golang.org/x/net v0.19.0 h1:zTwKpTd2XuCqf8huc7Fo2iSy+4RHPd10s4KzeTnVr1c= -golang.org/x/net v0.19.0/go.mod h1:CfAk/cbD4CthTvqiEl8NpboMuiuOYsAr/7NOjZJtv1U= golang.org/x/net v0.20.0 h1:aCL9BSgETF1k+blQaYUBx9hJ9LOGP3gAVemcZlf1Kpo= golang.org/x/net v0.20.0/go.mod h1:z8BVo6PvndSri0LbOE3hAn0apkU+1YvI6E70E9jsnvY= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.2.0/go.mod h1:Cwn6afJ8jrQwYMxQDTpISoXmXW9I6qF6vDeuuoX3Ibs= -golang.org/x/oauth2 v0.14.0 h1:P0Vrf/2538nmC0H+pEQ3MNFRRnVR7RlqyVw+bvm26z0= -golang.org/x/oauth2 v0.14.0/go.mod h1:lAtNWgaWfL4cm7j2OV8TxGi9Qb7ECORx8DktCY74OwM= -golang.org/x/oauth2 v0.15.0 h1:s8pnnxNVzjWyrvYdFUQq5llS1PX2zhPXmccZv99h7uQ= -golang.org/x/oauth2 v0.15.0/go.mod h1:q48ptWNTY5XWf+JNten23lcvHpLJ0ZSxF5ttTHKVCAM= golang.org/x/oauth2 v0.16.0 h1:aDkGMBSYxElaoP81NpoUoz2oo2R2wHdZpGToUxfyQrQ= golang.org/x/oauth2 v0.16.0/go.mod h1:hqZ+0LWXsiVoZpeld6jVt06P3adbS2Uu911W1SsJv2o= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -492,10 +451,6 @@ golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.14.0 h1:Vz7Qs629MkJkGyHxUlRHizWJRG2j8fbQKjELVSNhy7Q= -golang.org/x/sys v0.14.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/sys v0.15.0 h1:h48lPFYpsTvQJZF4EKyI4aLHaev3CxivZmv7yZig9pc= -golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.16.0 h1:xWw16ngr6ZMtmxDyKyIgsE93KNKz5HKmMa3b8ALHidU= golang.org/x/sys v0.16.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= @@ -504,10 +459,6 @@ golang.org/x/term v0.2.0/go.mod h1:TVmDHMZPmdnySmBfhjOoOdhjzdE1h4u1VwSiw2l1Nuc= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= golang.org/x/term v0.13.0/go.mod h1:LTmsnFJwVN6bCy1rVCoS+qHT1HhALEFxKncY3WNNh4U= -golang.org/x/term v0.14.0 h1:LGK9IlZ8T9jvdy6cTdfKUCltatMFOehAQo9SRC46UQ8= -golang.org/x/term v0.14.0/go.mod h1:TySc+nGkYR6qt8km8wUhuFRTVSMIX3XPR58y2lC8vww= -golang.org/x/term v0.15.0 h1:y/Oo/a/q3IXu26lQgl04j/gjuBDOBlx7X6Om1j2CPW4= -golang.org/x/term v0.15.0/go.mod h1:BDl952bC7+uMoWR75FIrCDx79TPU9oHkTZ9yRbYOrX0= golang.org/x/term v0.16.0 h1:m+B6fahuftsE9qjo0VWp2FW0mB3MTJvR0BaMQrq0pmE= golang.org/x/term v0.16.0/go.mod h1:yn7UURbUtPyrVJPGPq404EukNFxcm/foM+bV/bfcDsY= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= diff --git a/pkg/models/model_external_reference.go b/pkg/models/model_external_reference.go index 133876af2..e65a0694a 100644 --- a/pkg/models/model_external_reference.go +++ b/pkg/models/model_external_reference.go @@ -1058,7 +1058,7 @@ func (scrapeRules ActorScraperConfig) getCustomRules() { Attribute: "attribute id you want, eg src for an image of href for a link", PostProcessing: []PostProcessing{{ Function: "builtin function to apply to the extarcted text, eg RegexString to extract with regex, Parse Date, lbs to kg, see postProcessing function for options. You may specify multiple function, eg RegexString to extract a Date followed by Parse Date if not in the right format", - Params: []string{`Paramerter depends on the functions requirements `}, + Params: []string{`Parameter depends on the functions requirements `}, }}, }) exampleConfig.GenericActorScrapingConfig["example scrape"] = siteDetails diff --git a/ui/src/locales/en-GB.json b/ui/src/locales/en-GB.json index f77d47886..408c663d9 100644 --- a/ui/src/locales/en-GB.json +++ b/ui/src/locales/en-GB.json @@ -126,7 +126,7 @@ "Search Fields": "Search Fields", "Defaults date range to the last week. Note:must match yyyy-mm-dd, include leading zeros": "Defaults date range to the last week. Note:must match yyyy-mm-dd, include leading zeros", "Scraper does not exist": "Scraper does not exist", - "Restart XBR to load new Sties": "Restart XBR to load new Sties", + "Restart XBVR to load new Sties": "Restart XBVR to load new Sties", "Scraper Url": "Scraper Url", "Name": "Name", "Company": "Company", @@ -153,7 +153,7 @@ "Scrape Site Actors after Scene Scrape": "Scrape Site Actors after Scene Scrape", "Stashdb Api Key":"Stashdb Api Key", "Scrape StashDB":"Scrape StashDB", - "Scrape Actor Detals from XBVR Sites":"Scrape Actor Detals from XBVR Sites", + "Scrape Actor Details from XBVR Sites":"Scrape Actor Details from XBVR Sites", "Enter Weight in kg":"Enter Weight in kg", "Enter Weight in lbs":"Enter Weight in lbs", "Height in feet": "Height in feet", @@ -209,8 +209,8 @@ "Limit Scraping":"Limit Scraping", "Leave blank, unless you want to link scenes from the new custom site to scenes on an existing studio site, e.g. VRHush on SLR or VRPorn to the main VRHush site": "Leave blank, unless you want to link scenes from the new custom site to scenes on an existing studio site, e.g. VRHush on SLR or VRPorn to the main VRHush site", "Display scene details":"Display scene details", - "Matching paramerters":"Matching paramerters", - "Days to wait after the release date, before linking. Useful where the main site releases after SLR/VRPorn/POVR, eg LethalHardware": "Days to wait after the release date, before linking. Useful where the main site releases after SLR/VRPorn/POVR, eg LethalHardware", + "Matching parameters":"Matching parameters", + "Days to wait after the release date, before linking. Useful where the main site releases after SLR/VRPorn/POVR, eg LethalHardcore": "Days to wait after the release date, before linking. Useful where the main site releases after SLR/VRPorn/POVR, eg LethalHardcore", "Number of days to keep re-linking scenes after the release date":"Number of days to keep re-linking scenes after the release date", "Keep Re-linking(days)":"Keep Re-linking(days)", "Delay linking(days)":"Delay linking(days)", @@ -220,7 +220,7 @@ "Boost Value":"Boost Value", "The number of days prior to the release date to match, eg if the scene release date is 23/05/2023 and the days prior is 3, it will search >= 20/05/2023. If days prior and after are 0, the range is not used":"The number of days prior to the release date to match, eg if the scene release date is 23/05/2023 and the days prior is 3, it will search >= 20/05/2023. If days prior and after are 0, the range is not used", "Days Prior":"Days Prior", - "The number of days after the release date to match, eg if the scene release date is 23/05/2023 and the days after is =3, it will search <= 23/05/2023. Usually set to 0. If days prior and after are 0, the range is not used":"The number of days after the release date to match, eg if the scene release date is 23/05/2023 and the days after is =3, it will search <= 23/05/2023. Usually set to 0. If days prior and after are 0, the range is not used", + "The number of days after the release date to match, eg if the scene release date is 23/05/2023 and the days after is 3, it will search <= 23/05/2023. Usually set to 0. If days prior and after are 0, the range is not used":"The number of days after the release date to match, eg if the scene release date is 23/05/2023 and the days after is 3, it will search <= 23/05/2023. Usually set to 0. If days prior and after are 0, the range is not used", "Days After":"Days After", "Exact Match Boost Value":"Exact Match Boost Value", "Individual Word Match Boost Value":"Individual Word Match Boost Value", @@ -233,11 +233,12 @@ "Scenes from Akternate Sites will be matched after Scene Scraping":"Scenes from Akternate Sites will be matched after Scene Scraping", "If a file is not matched to a scene, then try scenes from Alternate Sites":"If a file is not matched to a scene, then try scenes from Alternate Sites", "When filtering for Scenes with Scripts or sorting by Script Published Date, scenes from Alternate Sites will be included. Note: Slows these queries":"When filtering for Scenes with Scripts or sorting by Script Published Date, scenes from Alternate Sites will be included. Note: Slows these queries", - "Do not link scenes prior to the specified date. The quality of metadata of older scenes is often poor and causes mismatches":"Do not link scenes prior to the specified date. The quality of metadata of older scenes is often poor and causes mismatches", "Remove Scene Links":"Remove Scene Links", "Remove Scene Links (Keep edits)":"Remove Scene Links (Keep edits)", "Delete scraped scenes":"Delete scraped scenes", "Link Scenes":"Link Scenes", "Linked to Alternate Sites":"Linked to Alternate Sites", - "Go": "Go" + "Go": "Go", + "Limit scraping to newest scenes on the website. Turn off if you are missing scenes.": "Limit scraping to newest scenes on the website. Turn off if you are missing scenes.", + "Highlights this studio in the scene view and includes scenes in the "Has subscription" attribute filter": "Highlights this studio in the scene view and includes scenes in the "Has subscription" attribute filter" } diff --git a/ui/src/views/options/overlays/SceneMatchParams.vue b/ui/src/views/options/overlays/SceneMatchParams.vue index 5bbdcdb1c..06ab7546f 100644 --- a/ui/src/views/options/overlays/SceneMatchParams.vue +++ b/ui/src/views/options/overlays/SceneMatchParams.vue @@ -7,7 +7,7 @@
-

{{ $t("Matching paramerters")}}: {{ site.name }}

+

{{ $t("Matching parameters")}}: {{ site.name }}

@@ -17,7 +17,7 @@
- @@ -88,7 +88,7 @@
- diff --git a/ui/src/views/options/sections/InterfaceAdvanced.vue b/ui/src/views/options/sections/InterfaceAdvanced.vue index 4822c5144..c45b19a69 100644 --- a/ui/src/views/options/sections/InterfaceAdvanced.vue +++ b/ui/src/views/options/sections/InterfaceAdvanced.vue @@ -64,7 +64,7 @@ - {{ $t('Scrape Actor Detals from XBVR Sites') }} + {{ $t('Scrape Actor Details from XBVR Sites') }} Save @@ -101,7 +101,7 @@
- + Save diff --git a/ui/src/views/options/sections/OptionsSceneDataScrapers.vue b/ui/src/views/options/sections/OptionsSceneDataScrapers.vue index a3989d29f..3b22e411d 100644 --- a/ui/src/views/options/sections/OptionsSceneDataScrapers.vue +++ b/ui/src/views/options/sections/OptionsSceneDataScrapers.vue @@ -41,10 +41,14 @@ + + + +
@@ -89,6 +93,7 @@
+ {{$t('Toggle Limit Scraping of all visible sites')}} {{$t('Toggle Subscriptions of all visible sites')}}
@@ -308,6 +313,15 @@ export default { } this.isLoading=false }, + async toggleAllLimitScraping(){ + const table = this.$refs.scraperTable; + this.isLoading=true + for (let i=0; i