Skip to content

Commit

Permalink
General Url fetcher script for ms and ghaf
Browse files Browse the repository at this point in the history 
Signed-off-by: Enes Öztürk <[email protected]>
  • Loading branch information
@enesoztrk
enesoztrk committed Nov 14, 2024
1 parent d25dc39 commit 2deb298
Show file tree
Hide file tree
Showing 3 changed files with 205 additions and 211 deletions.
248 changes: 113 additions & 135 deletions modules/reference/services/proxy-server/3proxy-config.nix
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,14 +9,26 @@
let
cfg = config.ghaf.reference.services.proxy-server;
inherit (lib) mkEnableOption mkIf;

proxyUserName = "proxy-user";
proxyGroupName = "proxy-admin";
proxyAllowListName = "allowlist.txt";
proxyWritableAllowListPath = "/etc/${proxyAllowListName}";
ms-url-fetcher = pkgs.callPackage ./ms_url_fetcher.nix {
allowListPath = proxyWritableAllowListPath;
};
url-fetcher = pkgs.callPackage ./url_fetcher.nix { };

msUrls = "https://endpoints.office.com/endpoints/worldwide?clientrequestid=b10c5ed1-bad1-445f-b386-b919946339a7";
ssrcUrls = "https://raw.githubusercontent.com/tiiuae/ghaf-rt-config/refs/heads/additional-urls/network/urls/ssrc_urls.json";
tiiUrls = "https://raw.githubusercontent.com/tiiuae/ghaf-rt-config/refs/heads/additional-urls/network/urls/tii_urls.json";
msExtraUrls = "https://raw.githubusercontent.com/tiiuae/ghaf-rt-config/refs/heads/additional-urls/network/urls/ms_extra_urls.json";

msAllowFilePath = "3proxy/ms_whitelist.txt";
ssrcAllowFilePath = "3proxy/ssrc_whitelist.txt";
tiiAllowFilePath = "3proxy/tii_whitelist.txt";
msExtraAllowFilePath = "3proxy/ms_extra_whitelist.txt";
# List of allow list paths
allowListPaths = [
msAllowFilePath
ssrcAllowFilePath
tiiAllowFilePath
msExtraAllowFilePath
];

_3proxy-restart = pkgs.writeShellApplication {
name = "3proxy-restart";
Expand All @@ -40,90 +52,47 @@ let
echo "3proxy service successfully started"
'';
};
tiiUrls = [
#for jira avatars
"*.gravatar.com"
# for confluence icons
"*.atlassian.com"
"*tii.ae"
"*tii.org"
"tiiuae.sharepoint.com"
"tiiuae-my.sharepoint.com"
"hcm22.sapsf.com"
"aderp.addigital.gov.ae"
"s1.mn1.ariba.com"
"tii.sourcing.mn1.ariba.com"
"a1c7ohabl.accounts.ondemand.com"
"flpnwc-ojffapwnic.dispatcher.ae1.hana.ondemand.com"
"*.docusign.com"
"access.clarivate.com"
];

ssrcUrls = [
"*.cachix.org"
"vedenemo.dev"
"loki.ghaflogs.vedenemo.dev"
"ghaflogs.vedenemo.dev"
"himalia.vedenemo.dev"
];
ghaf_url_fetcher =
let
logTag = "ghaf-url-fetcher";
in
pkgs.writeShellApplication {
name = "ghaf-url-fetcher";
runtimeInputs = [
url-fetcher
pkgs.inetutils
];
text = ''
extraMsUrls = [
#ms366
"graph.microsoft.com"
"ocws.officeapps.live.com"
"microsoft365.com"
"*.azureedge.net" # microsoft365 icons
"consentreceiverfd-prod.azurefd.net" # ms365 cookies
"c.s-microsoft.com"
"js.monitor.azure.com"
"ocws.officeapps.live.com"
"northcentralus0-mediap.svc.ms"
"*.bing.com"
"cdnjs.cloudfare.com"
"store-images.s-microsoft.com"
"www.office.com"
"res-1.cdn.office.net"
"secure.skypeassets.com"
"js.live.net"
"skyapi.onedrive.live.com"
"am3pap006files.storage.live.com"
"c7rr5q.am.files.1drv.com"
#teams
"teams.live.com"
"*.teams.live.com"
"fpt.live.com" # teams related
"statics.teams.cdn.live.net"
"ipv6.login.live.com"
#outlook
"outlook.live.com" # outlook login
"csp.microsoft.com"
"arc.msn.com"
"www.msn.com"
"outlook.com"
#https://learn.microsoft.com/en-us/microsoft-365/enterprise/managing-office-365-endpoints?view=o365-worldwide#why-do-i-see-names-such-as-nsatcnet-or-akadnsnet-in-the-microsoft-domain-names
"*.akadns.net"
"*.akam.net"
"*.akamai.com"
"*.akamai.net"
"*.akamaiedge.net"
"*.akamaihd.net"
"*.akamaized.net"
"*.edgekey.net"
"*.edgesuite.net"
"*.nsatc.net"
"*.exacttarget.com"
#onedrive
"1drv.ms"
"onedrive.live.com"
"p.sfx.ms"
"my.microsoftpersonalcontent.com"
"*.onedrive.com"
"cdn.onenote.net"
"wvcyna.db.files.1drv.com"
"*.storage.live.com"
];
# Concatenate the lists and join with commas
concatenatedUrls = builtins.concatStringsSep "," (tiiUrls ++ ssrcUrls ++ extraMsUrls);
echo "Attempting to start ghaf URL fetcher script"
# Fetch SSRC URLs
if ! url-fetcher -u "${ssrcUrls}" -p "/etc/${ssrcAllowFilePath}"; then
logger -t "${logTag}" "Failed to fetch SSRC URLs"
exit 2
fi
sleep 1
# Fetch TII URLs
if ! url-fetcher -u "${tiiUrls}" -p "/etc/${tiiAllowFilePath}"; then
logger -t "${logTag}" "Failed to fetch TII URLs"
exit 2
fi
sleep 1
# Fetch extra Microsoft URLs
if ! url-fetcher -u "${msExtraUrls}" -p "/etc/${msExtraAllowFilePath}"; then
logger -t "${logTag}" "Failed to fetch extra Microsoft URLs"
exit 2
fi
exit 0 # Success exit code
'';
};

config_file_content = ''
# log to stdout
Expand All @@ -136,9 +105,11 @@ let
#private addresses
deny * * 0.0.0.0/8,127.0.0.0/8,10.0.0.0/8,100.64.0.0/10,172.16.0.0/12,192.168.0.0/16,::,::1,fc00::/7
allow * * ${concatenatedUrls} *
#include dynamic whitelist ips
include "${proxyWritableAllowListPath}"
include "/etc/${msAllowFilePath}"
include "/etc/${ssrcAllowFilePath}"
include "/etc/${tiiAllowFilePath}"
include "/etc/${msExtraAllowFilePath}"
deny * * * *
maxconn 200
Expand Down Expand Up @@ -177,14 +148,23 @@ in
group = "${proxyGroupName}";
};

# Set up the permissions for allowlist.txt
environment.etc."${proxyAllowListName}" = {
text = '''';
user = "${proxyUserName}"; # Owner is proxy-user
group = "${proxyGroupName}"; # Group is proxy-admin
mode = "0660"; # Permissions: read/write for owner/group, no permissions for others
};
# Apply the allowListConfig generated from the list

# Create environment.etc configuration for each allow list path
# Loop over the allowListPaths and apply the configuration directly
environment.etc = builtins.foldl' (
acc: path:
acc
// {
"${path}" = {
text = '''';
user = "${proxyUserName}"; # Owner is proxy-user
group = "${proxyGroupName}"; # Group is proxy-admin
mode = "0660"; # Permissions: read/write for owner/group, no permissions for others
};
}
) { } allowListPaths;
# Apply the configurations for each allow list path
# Allow proxy-admin group to manage specific systemd services without a password
security = {
polkit = {
Expand All @@ -207,7 +187,7 @@ in

};

environment.systemPackages = [ ms-url-fetcher ];
environment.systemPackages = [ url-fetcher ];
#Firewall Settings
networking = {
firewall.enable = true;
Expand All @@ -218,12 +198,12 @@ in
'';
};
# systemd service for fetching the file
systemd.services.fetchFile = {
description = "Fetch a file periodically with retries if internet is available";
systemd.services.msFetchUrl = {
description = "Fetch microsoft URLs periodically with retries if internet is available";

serviceConfig = {
ExecStart = "${ms-url-fetcher}/bin/ms-url-fetch";
# Ensure fetchFile starts after the network is up
ExecStart = "${url-fetcher}/bin/url-fetcher -u ${msUrls} -p /etc/${msAllowFilePath}";
# Ensure msFetchUrl starts after the network is up
Type = "simple";
# Retry until systemctl restart 3proxy succeeds
ExecStartPost = "${_3proxy-restart}/bin/3proxy-restart";
Expand All @@ -235,8 +215,8 @@ in
};

# systemd timer to trigger the service every 10 minutes
systemd.timers.fetchFile = {
description = "Run fetch-file periodically";
systemd.timers.msFetchUrl = {
description = "Run msFetchUrl periodically";
wantedBy = [ "timers.target" ];
timerConfig = {
User = "${proxyUserName}";
Expand All @@ -246,6 +226,35 @@ in
};
};

# systemd service for fetching the file
systemd.services.ghafFetchUrl = {
description = "Fetch ghaf related URLs periodically with retries if internet is available";

serviceConfig = {
ExecStart = "${ghaf_url_fetcher}/bin/ghaf-url-fetcher";
# Ensure ghafFetchUrl starts after the network is up
Type = "simple";
# Retry until systemctl restart 3proxy succeeds
ExecStartPost = "${_3proxy-restart}/bin/3proxy-restart";
# Restart policy on failure
Restart = "on-failure"; # Restart the service if it fails
RestartSec = "15s"; # Wait 15 seconds before restarting
User = "${proxyUserName}";
};
};

# systemd timer to trigger the service every 10 minutes
systemd.timers.ghafFetchUrl = {
description = "Run ghafFetchUrl periodically";
wantedBy = [ "timers.target" ];
timerConfig = {
User = "${proxyUserName}";
Persistent = true; # Ensures the timer runs after a system reboot
OnCalendar = "hourly"; # Set to your desired schedule
OnBootSec = "90s";
};
};

systemd.services."3proxy".serviceConfig = {
RestartSec = "5s";
User = "${proxyUserName}";
Expand All @@ -258,37 +267,6 @@ in
confFile = pkgs.writeText "3proxy.conf" ''
${config_file_content}
'';

/*
NOTE allow and deny configurations should must be placed before the other configs
it is not possible to do with extraConfig. Because it appends the file
*/
/*
services = [
{
type = "proxy";
bindAddress = "${netvmAddr}";
inherit (cfg) bindPort;
maxConnections = 200;
auth = [ "iponly" ];
acl = [
{
rule = "allow";
targets = tiiUrls;
}
{
rule = "allow";
targets = ssrcUrls;
}
{
rule = "allow";
targets = extraMsUrls;
}
{ rule = "deny"; }
];
}
];
*/
};

};
Expand Down
76 changes: 0 additions & 76 deletions modules/reference/services/proxy-server/ms_url_fetcher.nix
View file

This file was deleted.

Loading

0 comments on commit 2deb298

Please sign in to comment.