Skip to content

Commit

Permalink
* fix every existing entity will be modified state since prop `RowVer…
Browse files Browse the repository at this point in the history
…sionedEntity.Version` will always be reset to `0`

* narrow generic constraint for `TEntity` from `class` to `RowVersionedEntity`
* rename param `existingOrNewLookup` to `isExistingEntityLookup`
@ `SaverWithRevision.SaveEntitiesWithRevision()`

* rename primary ctor param `registeredLocksLookup` to `registeredLocksKeyByType` @ RetryCrawlWorker.cs
- invokes to `.ForNoKeyUpdateHint()` following previous commit 97d426a @ `AuthorRevisionSaver.SaveAuthorRevisions()`
* inline `using LinqKit` @ (Sub)ReplySaver.cs
@ c#/crawler
  • Loading branch information
n0099 committed May 18, 2024
1 parent 6870be0 commit 0f7ff83
Show file tree
Hide file tree
Showing 5 changed files with 10 additions and 15 deletions.
1 change: 0 additions & 1 deletion c#/crawler/src/Tieba/Crawl/Saver/AuthorRevisionSaver.cs
Original file line number Diff line number Diff line change
Expand Up @@ -55,7 +55,6 @@ public class AuthorRevisionSaver(
.Select(latestRevisionProjectionFactory)
.AsCte() // https://stackoverflow.com/questions/49854322/usage-of-for-update-in-window-function-postgres#comment86726589_49854322
.Where(e => e.Rank == 1)
.AsPostgreSQL().ForNoKeyUpdateHint()
.ToLinqToDB().AsEnumerable()
.Join(posts, e => e.Uid, p => p.AuthorUid, (e, p) =>
(
Expand Down
4 changes: 1 addition & 3 deletions c#/crawler/src/Tieba/Crawl/Saver/Post/ReplySaver.cs
Original file line number Diff line number Diff line change
@@ -1,5 +1,3 @@
using PredicateBuilder = LinqKit.PredicateBuilder;

namespace tbm.Crawler.Tieba.Crawl.Saver.Post;

public class ReplySaver(
Expand Down Expand Up @@ -49,7 +47,7 @@ public override SaverChangeSet<ReplyPost> Save(CrawlerDbContext db)
{
var changeSet = Save(db, r => r.Pid,
r => new ReplyRevision {TakenAt = r.UpdatedAt ?? r.CreatedAt, Pid = r.Pid},
PredicateBuilder.New<ReplyPost>(r => Posts.Keys.Contains(r.Pid)));
LinqKit.PredicateBuilder.New<ReplyPost>(r => Posts.Keys.Contains(r.Pid)));

replyContentImageSaver.Save(db, changeSet.NewlyAdded);
PostSaveHandlers += AuthorRevisionSaver.SaveAuthorExpGradeRevisions(db, changeSet.AllAfter).Invoke;
Expand Down
4 changes: 1 addition & 3 deletions c#/crawler/src/Tieba/Crawl/Saver/Post/SubReplySaver.cs
Original file line number Diff line number Diff line change
@@ -1,5 +1,3 @@
using LinqKit;

namespace tbm.Crawler.Tieba.Crawl.Saver.Post;

public class SubReplySaver(
Expand Down Expand Up @@ -44,7 +42,7 @@ public override SaverChangeSet<SubReplyPost> Save(CrawlerDbContext db)
{
var changeSet = Save(db, sr => sr.Spid,
sr => new SubReplyRevision {TakenAt = sr.UpdatedAt ?? sr.CreatedAt, Spid = sr.Spid},
PredicateBuilder.New<SubReplyPost>(sr => Posts.Keys.Contains(sr.Spid)));
LinqKit.PredicateBuilder.New<SubReplyPost>(sr => Posts.Keys.Contains(sr.Spid)));
PostSaveHandlers += AuthorRevisionSaver.SaveAuthorExpGradeRevisions(db, changeSet.AllAfter).Invoke;

return changeSet;
Expand Down
12 changes: 6 additions & 6 deletions c#/crawler/src/Tieba/Crawl/Saver/SaverWithRevision.cs
Original file line number Diff line number Diff line change
Expand Up @@ -25,21 +25,21 @@ public abstract partial class SaverWithRevision<TBaseRevision>
protected void SaveEntitiesWithRevision<TEntity, TRevision>(
CrawlerDbContext db,
Func<TEntity, TRevision> revisionFactory,
ILookup<bool, TEntity> existingOrNewLookup,
ILookup<bool, TEntity> isExistingEntityLookup,
Func<TEntity, TEntity> existingSelector,
UserSaver.FieldChangeIgnorance? userFieldUpdateIgnorance = null,
UserSaver.FieldChangeIgnorance? userFieldRevisionIgnorance = null)
where TEntity : class
where TEntity : RowVersionedEntity
where TRevision : BaseRevisionWithSplitting
{
db.Set<TEntity>().AddRange(existingOrNewLookup[false]); // newly added
var newRevisions = existingOrNewLookup[true].Select(newEntity =>
db.Set<TEntity>().AddRange(isExistingEntityLookup[false]); // newly added
var newRevisions = isExistingEntityLookup[true].Select(newEntity =>
{
var entityInTracking = existingSelector(newEntity);
var entityEntry = db.Entry(entityInTracking);
// this will mutate existingEntity which is referenced by entry
entityEntry.CurrentValues.SetValues(newEntity);
entityEntry.CurrentValues.SetValues(newEntity); // mutate existingEntity that referenced by entry
entityEntry.Property(e => e.Version).IsModified = false; // newEntity.Version will always be default 0
bool IsTimestampingFieldName(string name) => name is nameof(BasePost.LastSeenAt)
or nameof(TimestampedEntity.CreatedAt) or nameof(TimestampedEntity.UpdatedAt);
Expand Down
4 changes: 2 additions & 2 deletions c#/crawler/src/Worker/RetryCrawlWorker.cs
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@ namespace tbm.Crawler.Worker;

public class RetryCrawlWorker(
ILogger<RetryCrawlWorker> logger,
IIndex<CrawlerLocks.Type, CrawlerLocks> registeredLocksLookup,
IIndex<CrawlerLocks.Type, CrawlerLocks> registeredLocksKeyByType,
CrawlPost crawlPost,
Func<Owned<CrawlerDbContext.NewDefault>> dbContextDefaultFactory,
Func<Owned<ThreadLateCrawlFacade.New>> threadLateCrawlFacadeFactory,
Expand All @@ -16,7 +16,7 @@ protected override async Task DoWork(CancellationToken stoppingToken)
foreach (var lockType in Enum.GetValues<CrawlerLocks.Type>())
{
if (stoppingToken.IsCancellationRequested) return;
var failed = registeredLocksLookup[lockType].RetryAllFailed();
var failed = registeredLocksKeyByType[lockType].RetryAllFailed();
if (failed.Count == 0) continue; // skip current lock type if there's nothing needs to retry
if (lockType == CrawlerLocks.Type.ThreadLate)
{
Expand Down

0 comments on commit 0f7ff83

Please sign in to comment.