Skip to content
This repository has been archived by the owner on Jul 3, 2020. It is now read-only.

Commit

Permalink
fix tag checker
Browse files Browse the repository at this point in the history
  • Loading branch information
@OpportunityLiu
OpportunityLiu committed Mar 7, 2020
1 parent 6936179 commit ac52931
Show file tree
Hide file tree
Showing 4 changed files with 122 additions and 84 deletions.
74 changes: 2 additions & 72 deletions EhDbReleaseBuilder/GitHubApiClient.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -48,6 +48,8 @@ public class HeadData
public JRaw Data { get; set; }
}

public Task CheckAsync(Namespace checkTags) => TagChecker.CheckAsync(_Database, checkTags);

private class FullUploadData : UploadData
{
public HeadData Head { get; set; }
Expand All @@ -59,78 +61,6 @@ public void Normalize()
_Database.Save();
}

private void _LogFailed(RecordDictionary db, string key, Record value, Namespace newNs, string newKey)
{
db.AddOrReplace(key, new Record(value.Name.Raw, value.Intro.Raw, value.Links.Raw
+ $"\nNow should be {newNs}:{newKey}"));
Console.WriteLine(" -> Failed");
}

public async Task CheckAsync(Namespace checkTags)
{
try
{
// skip rows & reclass
foreach (var db in _Database.Values.Where(v => v.Namespace > Namespace.Reclass && checkTags.HasFlag(v.Namespace)))
{
var ns = db.Namespace;
Console.WriteLine($"Checking namespace {ns} with {db.RawData.Count} items");
for (var i = 0; i < db.RawData.Count; i++)
{
var data = db.RawData[i];
Console.Write($"[{i,6}/{db.RawData.Count}] {data.Key}");
if (string.IsNullOrWhiteSpace(data.Key))
{
Console.WriteLine(" -> Skipped empty tag");
continue;
}
var (newNs, newKey) = await TagChecker.CheckAsync(ns, data.Key);
if (newKey is null)
{
db.Remove(data.Key, false);
Console.WriteLine(" -> Delete");
}
else if (newNs != ns)
{
Console.Write($" -> Move to {newNs}:{newKey}");
try
{
_Database[newNs].Add(newKey, data.Value);
db.Remove(data.Key, true);
Console.WriteLine(" -> Succeed");
}
catch
{
_LogFailed(db, data.Key, data.Value, newNs, newKey);
}
}
else if (newKey != data.Key)
{
Console.Write($" -> Rename to {newKey}");
try
{
db.Rename(data.Key, newKey);
Console.WriteLine(" -> Succeed");
}
catch
{
_LogFailed(db, data.Key, data.Value, newNs, newKey);
}
}
else
{
Console.WriteLine(" -> Valid");
}
}
_Database.Save();
}
}
finally
{
_Database.Save();
}
}

public void Publish()
{
foreach (var item in Directory.CreateDirectory(_Target).GetFiles())
Expand Down
2 changes: 1 addition & 1 deletion EhDbReleaseBuilder/Program.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@ public static async Task Main(string source, string target, Namespace checkTags
Console.WriteLine($@"EhDbReleaseBuilder started.
Source: {source}
Target: {target}
Check tags: {checkTags}
Check tags: {(checkTags > Namespace.Reclass ? checkTags.ToString() : "<Disabled>")}
");
var client = new GitHubApiClient(source, target);
if (checkTags > Namespace.Reclass)
Expand Down
2 changes: 1 addition & 1 deletion EhDbReleaseBuilder/Properties/launchSettings.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@
"profiles": {
"EhDbReleaseBuilder": {
"commandName": "Project",
"commandLineArgs": "--source C:/Users/lzy/Documents/Source/EhTagTranslation/EhTagConnector/EHTT-DB/ --target C:/Users/lzy/Documents/Source/EhTagTranslation/EhTagConnector/EHTT-DB/publish --check-tags Misc,Male,Female,Parody,Character,Group,Artist"
"commandLineArgs": "--source C:/Users/lzy/Documents/Source/EhTagTranslation/EhTagConnector/EHTT-DB/ --target C:/Users/lzy/Documents/Source/EhTagTranslation/EhTagConnector/EHTT-DB/publish --check-tags Parody,Character,Group,Artist"
}
}
}
128 changes: 118 additions & 10 deletions EhDbReleaseBuilder/TagChecker.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -51,7 +51,35 @@ private static Tag _FindCache(Namespace ns, string raw)
return tag;
}

public static async Task<(Namespace, string)> CheckAsync(Namespace ns, string raw)
private static async Task<TagSuggest> _PostApiAsync(Namespace? ns, string raw)
{
var text = raw.Length > 50 ? raw.Substring(0, 50) : raw;
if (ns != null)
text = ns + ":" + raw;
var response = await _HttpClient.PostAsync("https://api.e-hentai.org/api.php", new StringContent(JsonConvert.SerializeObject(new
{
method = "tagsuggest",
text,
})));
var resultStr = await response.Content.ReadAsStringAsync();
if (resultStr.Contains("{\"tags\":[]}"))
{
if (raw.Contains('.'))
{
return await _PostApiAsync(ns, raw.Substring(0, raw.IndexOf('.') - 1));
}
else
{
return null;
}
}

var result = JsonConvert.DeserializeObject<TagSuggest>(resultStr);
_FillCache(result);
return result;
}

public static async Task<(Namespace, string)> _CheckTagAsync(Namespace ns, string raw)
{
if (raw.Length <= 2)
return (ns, raw);
Expand All @@ -60,22 +88,26 @@ private static Tag _FindCache(Namespace ns, string raw)

if (match is null)
{
var response = await _HttpClient.PostAsync("https://api.e-hentai.org/api.php", new StringContent(JsonConvert.SerializeObject(new
{
method = "tagsuggest",
text = $"{(raw.Length > 50 ? raw.Substring(0, 50) : raw)}",
})));
var resultStr = await response.Content.ReadAsStringAsync();
if (!resultStr.Contains("{\"tags\":[]}"))
var result = await _PostApiAsync(null, raw);
if (result != null)
{
var result = JsonConvert.DeserializeObject<TagSuggest>(resultStr);
_FillCache(result);
// check exact match first
match = result.tags.Values.FirstOrDefault(tag => tag.ns == ns && tag.tn == raw)
// find from misc ns and master in this ns
?? result.tags.Values.FirstOrDefault(tag => tag.ns == Namespace.Misc && tag.tn == raw && tag.mns == ns);
}
}

if (match is null)
{
var result = await _PostApiAsync(ns, raw);
if (result != null)
{
// check exact match first
match = result.tags.Values.FirstOrDefault(tag => tag.ns == ns && tag.tn == raw)
// find from misc ns and master in this ns
?? result.tags.Values.FirstOrDefault(tag => tag.ns == Namespace.Misc && tag.tn == raw && tag.mns == ns);
}
}

if (match is null)
Expand All @@ -84,5 +116,81 @@ private static Tag _FindCache(Namespace ns, string raw)
return (ns, raw);
return (match.mns ?? Namespace.Misc, match.mtn);
}

private static void _LogFailed(RecordDictionary db, string key, Record value, Namespace newNs, string newKey)
{
db.AddOrReplace(key, new Record(value.Name.Raw, value.Intro.Raw, value.Links.Raw
+ $"\nNow should be {newNs}:{newKey}"));
Console.WriteLine(" -> Failed");
}
private static async Task _CheckNsAsync(Database database, Namespace ns)
{
var db = database[ns];
Console.WriteLine($"Checking namespace {ns} with {db.RawData.Count} lines");
for (var i = 0; i < db.RawData.Count; i++)
{
var data = db.RawData[i];
Console.Write($" [{i,4}/{db.RawData.Count}] {data.Key}");
if (string.IsNullOrWhiteSpace(data.Key))
{
Console.WriteLine(" -> Skipped empty tag");
continue;
}
var (newNs, newKey) = await _CheckTagAsync(ns, data.Key);
if (newKey is null)
{
db.Remove(data.Key, false);
Console.WriteLine(" -> Delete");
}
else if (newNs != ns)
{
Console.Write($" -> Move to {newNs}:{newKey}");
try
{
database[newNs].Add(newKey, data.Value);
db.Remove(data.Key, true);
Console.WriteLine(" -> Succeed");
}
catch
{
_LogFailed(db, data.Key, data.Value, newNs, newKey);
}
}
else if (newKey != data.Key)
{
Console.Write($" -> Rename to {newKey}");
try
{
db.Rename(data.Key, newKey);
Console.WriteLine(" -> Succeed");
}
catch
{
_LogFailed(db, data.Key, data.Value, newNs, newKey);
}
}
else
{
Console.WriteLine(" -> Valid");
}
}
}

public static async Task CheckAsync(Database database, Namespace checkTags)
{
try
{
// skip rows & reclass
foreach (var ns in database.Keys.Where(v => v > Namespace.Reclass && checkTags.HasFlag(v)))
{
await _CheckNsAsync(database, ns);
database.Save();
}
}
finally
{
database.Save();
}
}
}
}

0 comments on commit ac52931

Please sign in to comment.