Skip to content

Commit e6a63fd

Browse files
iPromKnightiPromKnight
authoredApr 11, 2024··
Allow configuration of producer urls (#203)
* Allow configuration of urls in scrapers by mounting the scrapers.json file over the one in the container * version bump
·
v2.0.28v2.0.23
1 parent 02101ac commit e6a63fd

File tree

11 files changed

+54
-34
lines changed

11 files changed

+54
-34
lines changed
 

‎deployment/docker/docker-compose.yaml

+7-7
Original file line numberDiff line numberDiff line change
@@ -94,7 +94,7 @@ services:
9494
condition: service_healthy
9595
env_file: stack.env
9696
hostname: knightcrawler-addon
97-
image: gabisonfire/knightcrawler-addon:2.0.22
97+
image: gabisonfire/knightcrawler-addon:2.0.23
9898
labels:
9999
logging: promtail
100100
networks:
@@ -117,7 +117,7 @@ services:
117117
redis:
118118
condition: service_healthy
119119
env_file: stack.env
120-
image: gabisonfire/knightcrawler-consumer:2.0.22
120+
image: gabisonfire/knightcrawler-consumer:2.0.23
121121
labels:
122122
logging: promtail
123123
networks:
@@ -138,7 +138,7 @@ services:
138138
redis:
139139
condition: service_healthy
140140
env_file: stack.env
141-
image: gabisonfire/knightcrawler-debrid-collector:2.0.22
141+
image: gabisonfire/knightcrawler-debrid-collector:2.0.23
142142
labels:
143143
logging: promtail
144144
networks:
@@ -152,7 +152,7 @@ services:
152152
migrator:
153153
condition: service_completed_successfully
154154
env_file: stack.env
155-
image: gabisonfire/knightcrawler-metadata:2.0.22
155+
image: gabisonfire/knightcrawler-metadata:2.0.23
156156
networks:
157157
- knightcrawler-network
158158
restart: "no"
@@ -163,7 +163,7 @@ services:
163163
postgres:
164164
condition: service_healthy
165165
env_file: stack.env
166-
image: gabisonfire/knightcrawler-migrator:2.0.22
166+
image: gabisonfire/knightcrawler-migrator:2.0.23
167167
networks:
168168
- knightcrawler-network
169169
restart: "no"
@@ -182,7 +182,7 @@ services:
182182
redis:
183183
condition: service_healthy
184184
env_file: stack.env
185-
image: gabisonfire/knightcrawler-producer:2.0.22
185+
image: gabisonfire/knightcrawler-producer:2.0.23
186186
labels:
187187
logging: promtail
188188
networks:
@@ -207,7 +207,7 @@ services:
207207
deploy:
208208
replicas: ${QBIT_REPLICAS:-0}
209209
env_file: stack.env
210-
image: gabisonfire/knightcrawler-qbit-collector:2.0.22
210+
image: gabisonfire/knightcrawler-qbit-collector:2.0.23
211211
labels:
212212
logging: promtail
213213
networks:

‎deployment/docker/src/components/knightcrawler.yaml

+7-7
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,7 @@ x-depends: &knightcrawler-app-depends
2020

2121
services:
2222
metadata:
23-
image: gabisonfire/knightcrawler-metadata:2.0.22
23+
image: gabisonfire/knightcrawler-metadata:2.0.23
2424
env_file: ../../.env
2525
networks:
2626
- knightcrawler-network
@@ -30,7 +30,7 @@ services:
3030
condition: service_completed_successfully
3131

3232
migrator:
33-
image: gabisonfire/knightcrawler-migrator:2.0.22
33+
image: gabisonfire/knightcrawler-migrator:2.0.23
3434
env_file: ../../.env
3535
networks:
3636
- knightcrawler-network
@@ -40,30 +40,30 @@ services:
4040
condition: service_healthy
4141

4242
addon:
43-
image: gabisonfire/knightcrawler-addon:2.0.22
43+
image: gabisonfire/knightcrawler-addon:2.0.23
4444
<<: [*knightcrawler-app, *knightcrawler-app-depends]
4545
restart: unless-stopped
4646
hostname: knightcrawler-addon
4747
ports:
4848
- "7000:7000"
4949

5050
consumer:
51-
image: gabisonfire/knightcrawler-consumer:2.0.22
51+
image: gabisonfire/knightcrawler-consumer:2.0.23
5252
<<: [*knightcrawler-app, *knightcrawler-app-depends]
5353
restart: unless-stopped
5454

5555
debridcollector:
56-
image: gabisonfire/knightcrawler-debrid-collector:2.0.22
56+
image: gabisonfire/knightcrawler-debrid-collector:2.0.23
5757
<<: [*knightcrawler-app, *knightcrawler-app-depends]
5858
restart: unless-stopped
5959

6060
producer:
61-
image: gabisonfire/knightcrawler-producer:2.0.22
61+
image: gabisonfire/knightcrawler-producer:2.0.23
6262
<<: [*knightcrawler-app, *knightcrawler-app-depends]
6363
restart: unless-stopped
6464

6565
qbitcollector:
66-
image: gabisonfire/knightcrawler-qbit-collector:2.0.22
66+
image: gabisonfire/knightcrawler-qbit-collector:2.0.23
6767
<<: [*knightcrawler-app, *knightcrawler-app-depends]
6868
restart: unless-stopped
6969
depends_on:

‎src/producer/src/Configuration/scrapers.json

+12-5
Original file line numberDiff line numberDiff line change
@@ -4,27 +4,34 @@
44
{
55
"Name": "SyncEzTvJob",
66
"IntervalSeconds": 60,
7-
"Enabled": true
7+
"Enabled": true,
8+
"Url": "https://eztv1.xyz/ezrss.xml",
9+
"XmlNamespace": "http://xmlns.ezrss.it/0.1/"
810
},
911
{
1012
"Name": "SyncNyaaJob",
1113
"IntervalSeconds": 60,
12-
"Enabled": true
14+
"Enabled": true,
15+
"Url": "https://nyaa.si/?page=rss&c=1_2&f=0",
16+
"XmlNamespace": "https://nyaa.si/xmlns/nyaa"
1317
},
1418
{
1519
"Name": "SyncTpbJob",
1620
"IntervalSeconds": 60,
17-
"Enabled": true
21+
"Enabled": true,
22+
"Url": "https://apibay.org/precompiled/data_top100_recent.json"
1823
},
1924
{
2025
"Name": "SyncYtsJob",
2126
"IntervalSeconds": 60,
22-
"Enabled": true
27+
"Enabled": true,
28+
"Url": "https://yts.am/rss"
2329
},
2430
{
2531
"Name": "SyncTgxJob",
2632
"IntervalSeconds": 60,
27-
"Enabled": true
33+
"Enabled": true,
34+
"Url": "https://tgx.rs/rss"
2835
},
2936
{
3037
"Name": "SyncDmmJob",

‎src/producer/src/Features/CrawlerSupport/BaseJsonCrawler.cs

+6
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,12 @@ public abstract class BaseJsonCrawler(IHttpClientFactory httpClientFactory, ILog
66

77
protected virtual async Task Execute(string collectionName)
88
{
9+
if (string.IsNullOrWhiteSpace(Url))
10+
{
11+
logger.LogWarning("No URL provided for {Source} crawl", Source);
12+
return;
13+
}
14+
915
logger.LogInformation("Starting {Source} crawl", Source);
1016

1117
using var client = httpClientFactory.CreateClient("Scraper");

‎src/producer/src/Features/CrawlerSupport/BaseXmlCrawler.cs

+6
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,12 @@ public abstract class BaseXmlCrawler(IHttpClientFactory httpClientFactory, ILogg
44
{
55
public override async Task Execute()
66
{
7+
if (string.IsNullOrWhiteSpace(Url))
8+
{
9+
logger.LogWarning("No URL provided for {Source} crawl", Source);
10+
return;
11+
}
12+
713
logger.LogInformation("Starting {Source} crawl", Source);
814

915
using var client = httpClientFactory.CreateClient(Literals.CrawlerClient);

‎src/producer/src/Features/CrawlerSupport/Scraper.cs

+4
Original file line numberDiff line numberDiff line change
@@ -7,4 +7,8 @@ public class Scraper
77
public int IntervalSeconds { get; set; } = 60;
88

99
public bool Enabled { get; set; } = true;
10+
11+
public string? Url { get; set; }
12+
13+
public string? XmlNamespace { get; set; }
1014
}

‎src/producer/src/Features/Crawlers/EzTv/EzTvCrawler.cs

+3-4
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,10 @@
11
namespace Producer.Features.Crawlers.EzTv;
22

3-
public class EzTvCrawler(IHttpClientFactory httpClientFactory, ILogger<EzTvCrawler> logger, IDataStorage storage) : BaseXmlCrawler(httpClientFactory, logger, storage)
3+
public class EzTvCrawler(IHttpClientFactory httpClientFactory, ILogger<EzTvCrawler> logger, IDataStorage storage, ScrapeConfiguration scrapeConfiguration) : BaseXmlCrawler(httpClientFactory, logger, storage)
44
{
5-
protected override string Url => "https://eztv1.xyz/ezrss.xml";
5+
protected override string Url => scrapeConfiguration.Scrapers.FirstOrDefault(x => x.Name.Equals("SyncEzTvJob", StringComparison.OrdinalIgnoreCase))?.Url ?? string.Empty;
66
protected override string Source => "EZTV";
7-
8-
private static readonly XNamespace XmlNamespace = "http://xmlns.ezrss.it/0.1/";
7+
private XNamespace XmlNamespace => scrapeConfiguration.Scrapers.FirstOrDefault(x => x.Name.Equals("SyncEzTvJob", StringComparison.OrdinalIgnoreCase))?.Url ?? string.Empty;
98

109
protected override IReadOnlyDictionary<string, string> Mappings =>
1110
new Dictionary<string, string>

‎src/producer/src/Features/Crawlers/Nyaa/NyaaCrawler.cs

+3-4
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,10 @@
11
namespace Producer.Features.Crawlers.Nyaa;
22

3-
public class NyaaCrawler(IHttpClientFactory httpClientFactory, ILogger<NyaaCrawler> logger, IDataStorage storage) : BaseXmlCrawler(httpClientFactory, logger, storage)
3+
public class NyaaCrawler(IHttpClientFactory httpClientFactory, ILogger<NyaaCrawler> logger, IDataStorage storage, ScrapeConfiguration scrapeConfiguration) : BaseXmlCrawler(httpClientFactory, logger, storage)
44
{
5-
protected override string Url => "https://nyaa.si/?page=rss&c=1_2&f=0";
5+
protected override string Url => scrapeConfiguration.Scrapers.FirstOrDefault(x => x.Name.Equals("SyncNyaaJob", StringComparison.OrdinalIgnoreCase))?.Url ?? string.Empty;
66
protected override string Source => "Nyaa";
7-
8-
private static readonly XNamespace XmlNamespace = "https://nyaa.si/xmlns/nyaa";
7+
private XNamespace XmlNamespace => scrapeConfiguration.Scrapers.FirstOrDefault(x => x.Name.Equals("SyncNyaaJob", StringComparison.OrdinalIgnoreCase))?.Url ?? string.Empty;
98

109
protected override IReadOnlyDictionary<string, string> Mappings =>
1110
new Dictionary<string, string>

‎src/producer/src/Features/Crawlers/Tgx/TgxCrawler.cs

+2-2
Original file line numberDiff line numberDiff line change
@@ -1,13 +1,13 @@
11
namespace Producer.Features.Crawlers.Tgx;
22

3-
public partial class TgxCrawler(IHttpClientFactory httpClientFactory, ILogger<TgxCrawler> logger, IDataStorage storage) : BaseXmlCrawler(httpClientFactory, logger, storage)
3+
public partial class TgxCrawler(IHttpClientFactory httpClientFactory, ILogger<TgxCrawler> logger, IDataStorage storage, ScrapeConfiguration scrapeConfiguration) : BaseXmlCrawler(httpClientFactory, logger, storage)
44
{
55
[GeneratedRegex(@"Size:\s+(.+?)\s+Added")]
66
private static partial Regex SizeStringExtractor();
77
[GeneratedRegex(@"(?i)\b(\d+(\.\d+)?)\s*([KMGT]?B)\b", RegexOptions.None, "en-GB")]
88
private static partial Regex SizeStringParser();
99

10-
protected override string Url => "https://tgx.rs/rss";
10+
protected override string Url => scrapeConfiguration.Scrapers.FirstOrDefault(x => x.Name.Equals("SyncTgxJob", StringComparison.OrdinalIgnoreCase))?.Url ?? string.Empty;
1111

1212
protected override string Source => "TorrentGalaxy";
1313
protected override IReadOnlyDictionary<string, string> Mappings

‎src/producer/src/Features/Crawlers/Tpb/TpbCrawler.cs

+2-2
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,8 @@
11
namespace Producer.Features.Crawlers.Tpb;
22

3-
public class TpbCrawler(IHttpClientFactory httpClientFactory, ILogger<TpbCrawler> logger, IDataStorage storage) : BaseJsonCrawler(httpClientFactory, logger, storage)
3+
public class TpbCrawler(IHttpClientFactory httpClientFactory, ILogger<TpbCrawler> logger, IDataStorage storage, ScrapeConfiguration scrapeConfiguration) : BaseJsonCrawler(httpClientFactory, logger, storage)
44
{
5-
protected override string Url => "https://apibay.org/precompiled/data_top100_recent.json";
5+
protected override string Url => scrapeConfiguration.Scrapers.FirstOrDefault(x => x.Name.Equals("SyncTpbJob", StringComparison.OrdinalIgnoreCase))?.Url ?? string.Empty;
66

77
protected override string Source => "TPB";
88

‎src/producer/src/Features/Crawlers/Yts/YtsCrawler.cs

+2-3
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,8 @@
11
namespace Producer.Features.Crawlers.Yts;
22

3-
public class YtsCrawler(IHttpClientFactory httpClientFactory, ILogger<YtsCrawler> logger, IDataStorage storage) : BaseXmlCrawler(httpClientFactory, logger, storage)
3+
public class YtsCrawler(IHttpClientFactory httpClientFactory, ILogger<YtsCrawler> logger, IDataStorage storage, ScrapeConfiguration scrapeConfiguration) : BaseXmlCrawler(httpClientFactory, logger, storage)
44
{
5-
protected override string Url => "https://yts.am/rss";
6-
5+
protected override string Url => scrapeConfiguration.Scrapers.FirstOrDefault(x => x.Name.Equals("SyncYtsJob", StringComparison.OrdinalIgnoreCase))?.Url ?? string.Empty;
76
protected override string Source => "YTS";
87
protected override IReadOnlyDictionary<string, string> Mappings
98
=> new Dictionary<string, string>

0 commit comments

Comments
 (0)
Please sign in to comment.