Skip to content

Commit

Permalink
revert: tracking migration
Browse files Browse the repository at this point in the history
  • Loading branch information
@MagicTheDev
MagicTheDev committed Jun 23, 2024
1 parent 9a4d3af commit 2b0147f
Show file tree
Hide file tree
Showing 2 changed files with 115 additions and 59 deletions.
44 changes: 23 additions & 21 deletions .idea/workspace.xml
View file

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

130 changes: 92 additions & 38 deletions gamewide/clan_verify/track.py
View file
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
import coc

import pendulum as pend
import ujson
import aiohttp
Expand All @@ -18,6 +18,60 @@



class League(Struct):
name : str

class ClanCapital(Struct):
capitalHallLevel: Optional[int] = 0

class Location(Struct):
name: str
id: int

class Members(Struct):
tag: str
name: str
expLevel: int
trophies: int
role: str
builderBaseTrophies: int
donations: int
donationsReceived: int
townHallLevel: int
league: League

class DBMember(Struct):
tag: str
name: str
expLevel: int
trophies: int
role: str
builderTrophies: int
donations: int
donationsReceived: int
townHallLevel: int
townhall: int
league: str


class Clan(Struct):
name: str
tag: str
type: str
clanLevel: int
isWarLogPublic: bool
members: int
clanPoints: int
clanCapitalPoints: int
capitalLeague: League
warLeague: League
warWinStreak: int
warWins: int
clanCapital: ClanCapital
memberList : List[Members]
location: Optional[Location] = None


async def fetch(url, session: aiohttp.ClientSession, headers):
async with session.get(url, headers=headers) as response:
if response.status == 200:
Expand All @@ -29,21 +83,11 @@ async def main():
config = ClanVerifyTrackingConfig()
db_client = MongoDatabase(stats_db_connection=config.stats_mongodb, static_db_connection=config.static_mongodb)

coc_client = coc.Client(raw_attribute=True)
keys: deque = await create_keys([config.coc_email.format(x=x) for x in range(config.min_coc_email, config.max_coc_email + 1)], [config.coc_password] * config.max_coc_email)
logger.info(f"{len(keys)} keys")
x = 1

changes = []
async for clan in db_client.basic_clan.find({}, {"tag" : 1, "changes" : 1}):
changes.append(UpdateOne({"_id" : clan.get("tag")}, {"$set" : {"changes" : clan.get("changes", {})}}, upsert=True))

await db_client.global_clans.bulk_write(changes, ordered=False)
print("UPDATED CHANGES")
return

while True:
#try:
try:
ranking_pipeline = [{"$unwind": "$memberList"},
{"$match": {"memberList.league": "Legend League"}},
{"$project": {"name": "$memberList.name", "tag": "$memberList.tag",
Expand All @@ -58,18 +102,16 @@ async def main():
{"$unset" : ["sort_field"]},
{"$out": {"db": "new_looper", "coll": "legend_rankings"}}
]
#await db_client.global_clans.aggregate(ranking_pipeline).to_list(length=None)
await db_client.global_clans.aggregate(ranking_pipeline).to_list(length=None)
logger.info("UPDATED RANKING")

keys = deque(keys)
if x % 20 == 0:
pipeline = [{"$match" : {"$or" : [{"members" : {"$lt" : 10}}, {"level" : {"$lt" : 3}}, {"capitalLeague" : "Unranked"}]}}, { "$group" : { "_id" : "$tag" } } ]
else:
pipeline = [{"$match": {"$nor" : [{"members" : {"$lt" : 10}}, {"level" : {"$lt" : 3}}, {"capitalLeague" : "Unranked"}]}}, {"$group": {"_id": "$tag"}}]

pipeline = [{"$match": {}}, {"$group": {"_id": "$tag"}}]
x += 1
all_tags = [x["_id"] for x in (await db_client.basic_clan.aggregate(pipeline).to_list(length=None))]
all_tags = [x["_id"] for x in (await db_client.global_clans.aggregate(pipeline).to_list(length=None))]
bot_clan_tags = await db_client.clans_db.distinct("tag")
all_tags = list(set(all_tags + bot_clan_tags))

Expand All @@ -78,7 +120,7 @@ async def main():
all_tags = [all_tags[i:i + size_break] for i in range(0, len(all_tags), size_break)]

for tag_group in all_tags:
#try:
try:
tasks = []
connector = TCPConnector(limit=500, enable_cleanup_closed=True)
timeout = ClientTimeout(total=1800)
Expand All @@ -94,22 +136,21 @@ async def main():

raid_week = gen_raid_date()
season = gen_season_date()
'''clan_group_members = await db_client.global_clans.find({"tag" : {"$in" : tag_group}}, {"tag" : 1, "_id" : 0, "data" : 1}).to_list(length=None)
clan_group_members = {x.get("data").get("tag") : x.get("memberList", []) for x in clan_group_members}'''
clan_group_members = await db_client.global_clans.find({"tag" : {"$in" : tag_group}}, {"tag" : 1, "_id" : 0, "memberList" : 1}).to_list(length=None)
clan_group_members = {x.get("tag") : x.get("memberList", []) for x in clan_group_members}
for response in responses: #type: bytes
# we shouldnt have completely invalid tags, they all existed at some point
if response is None:
continue

clan = ujson.loads(response)
if clan.get("members") == 0:
await db_client.deleted_clans.insert_one(clan)
#changes.append(DeleteOne({"tag": clan.tag}))
clan = decode(response, type=Clan)
if clan.members == 0:
await db_client.deleted_clans.insert_one(ujson.loads(response))
changes.append(DeleteOne({"tag": clan.tag}))
else:
clan = coc.Clan(data=clan, client=coc_client)
'''members = []
members = []
if clan.tag in clan_group_members:
clan_member_list = [m for m in clan_group_members.get(clan.tag)]
clan_member_list = [DBMember(**(m | {"townHallLevel" : m.get("townhall")})) for m in clan_group_members.get(clan.tag)]
new_joins = [player for player in clan.memberList if player.tag not in set(p.tag for p in clan_member_list)]
new_leaves = [player for player in clan_member_list if player.tag not in set(p.tag for p in clan.memberList)]
for join in new_joins:
Expand All @@ -135,29 +176,42 @@ async def main():
for member in clan.memberList:
members.append({"name": member.name, "tag" : member.tag, "role" : member.role, "expLevel" : member.expLevel, "trophies" : member.trophies,
"townhall" : member.townHallLevel, "league" : member.league.name,
"builderTrophies" : member.builderBaseTrophies, "donations" : member.donations, "donationsReceived" : member.donationsReceived})'''
changes.append(UpdateOne({"_id": clan.tag},
"builderTrophies" : member.builderBaseTrophies, "donations" : member.donations, "donationsReceived" : member.donationsReceived})
changes.append(UpdateOne({"tag": clan.tag},
{"$set":
{
"isValid" : clan.member_count >= 5,
f"changes.clanCapital.{raid_week}": {"trophies" : clan.capital_points, "league" : clan.capital_league.name},
f"changes.clanWarLeague.{season}": {"league": clan.war_league.name},
"data" : clan._raw_data
{"name": clan.name,
"members" : clan.members,
"level" : clan.clanLevel,
"type" : clan.type,
"location" : {"id" :clan.location.id if clan.location else clan.location, "name" : clan.location.name if clan.location else clan.location},
"clanCapitalPoints" : clan.clanCapitalPoints,
"clanPoints" : clan.clanPoints,
"capitalLeague" : clan.capitalLeague.name,
"warLeague" : clan.warLeague.name,
"warWinStreak" : clan.warWinStreak,
"warWins" : clan.warWins,
"clanCapitalHallLevel" : clan.clanCapital.capitalHallLevel,
"isValid" : clan.members >= 5,
"openWarLog" : clan.isWarLogPublic,
f"changes.clanCapital.{raid_week}": {"trophies" : clan.clanCapitalPoints, "league" : clan.capitalLeague.name},
f"changes.clanWarLeague.{season}": {"league": clan.warLeague.name},
"memberList": members
},
},
upsert=True))



if changes:
await db_client.global_clans.bulk_write(changes, ordered=False)
logger.info(f"Made {len(changes)} clan changes")

if join_leave_changes:
await db_client.join_leave_history.bulk_write(join_leave_changes, ordered=False)
logger.info(f"Made {len(join_leave_changes)} join/leave changes")
#except Exception:
#continue

#except Exception:
#continue
except Exception:
continue

except Exception:
continue

0 comments on commit 2b0147f

Please sign in to comment.