Skip to content

Commit

Permalink
* flip operands to make param newValue preceding oldValue @ (Sub)…
Browse files Browse the repository at this point in the history
…ReplySaver.cs

* split into two partial classes @ ReplySaver.cs
@ crawler

* reorder methods
* expand list with newline
@ c#
  • Loading branch information
n0099 committed May 15, 2024
1 parent a0f48f3 commit 8769dc0
Show file tree
Hide file tree
Showing 6 changed files with 36 additions and 31 deletions.
13 changes: 6 additions & 7 deletions c#/crawler/src/Tieba/Crawl/Crawler/ThreadCrawler.cs
Original file line number Diff line number Diff line change
Expand Up @@ -48,11 +48,10 @@ protected override IEnumerable<Request> GetRequestsForPage(Page page, Cancellati
];
}

protected ThreadRequest.Types.Data GetRequestDataForClientVersion602(Page page) =>
new()
{
Kw = forumName,
Pn = (int)page,
Rn = 30
};
protected ThreadRequest.Types.Data GetRequestDataForClientVersion602(Page page) => new()
{
Kw = forumName,
Pn = (int)page,
Rn = 30
};
}
8 changes: 7 additions & 1 deletion c#/crawler/src/Tieba/Crawl/Parser/UserParser.cs
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,13 @@ public partial class UserParser(ConcurrentDictionary<Uid, User> users)
var (portrait, portraitUpdatedAt) = ExtractPortrait(el.Portrait);
if (uid < 0) // historical anonymous user
{
return new() {Uid = uid, Name = el.NameShow, Portrait = portrait, PortraitUpdatedAt = portraitUpdatedAt};
return new()
{
Uid = uid,
Name = el.NameShow,
Portrait = portrait,
PortraitUpdatedAt = portraitUpdatedAt
};
}
// will be an empty string when the user hasn't set a username for their baidu account yet
Expand Down
28 changes: 15 additions & 13 deletions c#/crawler/src/Tieba/Crawl/Saver/Post/ReplySaver.cs
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@

namespace tbm.Crawler.Tieba.Crawl.Saver;

public class ReplySaver(
public partial class ReplySaver(
ILogger<ReplySaver> logger,
ConcurrentDictionary<PostId, ReplyPost> posts,
ReplySignatureSaver replySignatureSaver,
Expand All @@ -21,13 +21,13 @@ protected override bool FieldUpdateIgnorance

protected override bool UserFieldUpdateIgnorance(string propName, object? oldValue, object? newValue) => propName switch
{ // FansNickname in reply response will always be null
nameof(User.FansNickname) when oldValue is not null && newValue is null => true,
nameof(User.FansNickname) when newValue is null && oldValue is not null => true,
_ => false
};

protected override bool UserFieldRevisionIgnorance(string propName, object? oldValue, object? newValue) => propName switch
{ // user icon will be null after UserParser.ResetUsersIcon() get invoked
nameof(User.Icon) when oldValue is null && newValue is not null => true,
nameof(User.Icon) when newValue is not null && oldValue is null => true,
_ => false
};

Expand All @@ -51,6 +51,15 @@ protected override bool FieldUpdateIgnorance
}
};

[SuppressMessage("StyleCop.CSharp.SpacingRules", "SA1025:Code should not contain multiple whitespace in a row")]
protected override NullFieldsBitMask GetRevisionNullFieldBitMask(string fieldName) => fieldName switch
{
nameof(ReplyPost.IsFold) => 1 << 2,
nameof(ReplyPost.DisagreeCount) => 1 << 4,
nameof(ReplyPost.Geolocation) => 1 << 5,
_ => 0
};

public override SaverChangeSet<ReplyPost> Save(CrawlerDbContext db)
{
var changeSet = Save(db, r => r.Pid,
Expand All @@ -65,16 +74,9 @@ public override SaverChangeSet<ReplyPost> Save(CrawlerDbContext db)

return changeSet;
}

[SuppressMessage("StyleCop.CSharp.SpacingRules", "SA1025:Code should not contain multiple whitespace in a row")]
protected override NullFieldsBitMask GetRevisionNullFieldBitMask(string fieldName) => fieldName switch
{
nameof(ReplyPost.IsFold) => 1 << 2,
nameof(ReplyPost.DisagreeCount) => 1 << 4,
nameof(ReplyPost.Geolocation) => 1 << 5,
_ => 0
};

}
public partial class ReplySaver
{
private static void SaveReplyContentImages(CrawlerDbContext db, IEnumerable<ReplyPost> replies)
{
var pidAndImageList = (
Expand Down
6 changes: 3 additions & 3 deletions c#/crawler/src/Tieba/Crawl/Saver/Post/SubReplySaver.cs
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@ protected override bool UserFieldUpdateIgnorance
nameof(User.Icon) => true,

// FansNickname in sub reply response will always be null
nameof(User.FansNickname) when oldValue is not null && newValue is null => true,
nameof(User.FansNickname) when newValue is null && oldValue is not null => true,

// DisplayName in users embedded in sub replies from response will be the legacy nickname
nameof(User.DisplayName) => true,
Expand All @@ -40,6 +40,8 @@ protected override bool UserFieldUpdateIgnorance
}
};

protected override NullFieldsBitMask GetRevisionNullFieldBitMask(string fieldName) => 0;

public override SaverChangeSet<SubReplyPost> Save(CrawlerDbContext db)
{
var changeSet = Save(db, sr => sr.Spid,
Expand All @@ -52,6 +54,4 @@ public override SaverChangeSet<SubReplyPost> Save(CrawlerDbContext db)

return changeSet;
}

protected override NullFieldsBitMask GetRevisionNullFieldBitMask(string fieldName) => 0;
}
10 changes: 5 additions & 5 deletions c#/crawler/src/Tieba/Crawl/Saver/Post/ThreadSaver.cs
Original file line number Diff line number Diff line change
Expand Up @@ -67,11 +67,6 @@ protected override bool UserFieldUpdateIgnorance
}
};

public override SaverChangeSet<ThreadPost> Save(CrawlerDbContext db) =>
Save(db, th => th.Tid,
th => new ThreadRevision {TakenAt = th.UpdatedAt ?? th.CreatedAt, Tid = th.Tid},
PredicateBuilder.New<ThreadPost>(th => Posts.Keys.Contains(th.Tid)));

[SuppressMessage("StyleCop.CSharp.SpacingRules", "SA1025:Code should not contain multiple whitespace in a row")]
protected override NullFieldsBitMask GetRevisionNullFieldBitMask(string fieldName) => fieldName switch
{
Expand All @@ -86,4 +81,9 @@ protected override bool UserFieldUpdateIgnorance
nameof(ThreadPost.Geolocation) => 1 << 10,
_ => 0
};

public override SaverChangeSet<ThreadPost> Save(CrawlerDbContext db) =>
Save(db, th => th.Tid,
th => new ThreadRevision {TakenAt = th.UpdatedAt ?? th.CreatedAt, Tid = th.Tid},
PredicateBuilder.New<ThreadPost>(th => Posts.Keys.Contains(th.Tid)));
}
2 changes: 0 additions & 2 deletions c#/shared/src/Db/TbmDbContext.cs
Original file line number Diff line number Diff line change
Expand Up @@ -139,9 +139,7 @@ protected override void OnModelCreating(ModelBuilder b)
b.Entity<ReplyContentImage>().ToTable($"tbmc_f{fid}_reply_content_image");

protected virtual void OnConfiguringNpgsql(NpgsqlDbContextOptionsBuilder builder) { }

protected virtual void OnBuildingNpgsqlDataSource(NpgsqlDataSourceBuilder builder) { }

private Lazy<NpgsqlDataSource> GetNpgsqlDataSource(string? connectionString) =>
_dataSourceSingleton ??= new(() =>

Check failure on line 144 in c#/shared/src/Db/TbmDbContext.cs

View workflow job for this annotation

GitHub Actions / build (crawler)

Make the enclosing instance method 'static' or remove this set on the 'static' field. (https://rules.sonarsource.com/csharp/RSPEC-2696)

Check failure on line 144 in c#/shared/src/Db/TbmDbContext.cs

View workflow job for this annotation

GitHub Actions / build (crawler)

Make the enclosing instance method 'static' or remove this set on the 'static' field. (https://rules.sonarsource.com/csharp/RSPEC-2696)

Check failure on line 144 in c#/shared/src/Db/TbmDbContext.cs

View workflow job for this annotation

GitHub Actions / build (imagePipeline)

Make the enclosing instance method 'static' or remove this set on the 'static' field. (https://rules.sonarsource.com/csharp/RSPEC-2696)

Check failure on line 144 in c#/shared/src/Db/TbmDbContext.cs

View workflow job for this annotation

GitHub Actions / build (imagePipeline)

Make the enclosing instance method 'static' or remove this set on the 'static' field. (https://rules.sonarsource.com/csharp/RSPEC-2696)

Check failure on line 144 in c#/shared/src/Db/TbmDbContext.cs

View workflow job for this annotation

GitHub Actions / build (shared)

Make the enclosing instance method 'static' or remove this set on the 'static' field. (https://rules.sonarsource.com/csharp/RSPEC-2696)

Check failure on line 144 in c#/shared/src/Db/TbmDbContext.cs

View workflow job for this annotation

GitHub Actions / build (shared)

Make the enclosing instance method 'static' or remove this set on the 'static' field. (https://rules.sonarsource.com/csharp/RSPEC-2696)
{
Expand Down

0 comments on commit 8769dc0

Please sign in to comment.