Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Added Sentry CLI & Other minor Fix #13

Open
wants to merge 15 commits into
base: main
Choose a base branch
from
+169 −101
Open

Added Sentry CLI & Other minor Fix #13

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
15 commits
Select commit Hold shift + click to select a range
57248f2
Fix Variable Sets pagination to retrieve all pages
Praveenraj-K Dec 26, 2024
7a08bba
Fix Variable Sets pagination to retrieve all pages
Praveenraj-K Dec 27, 2024
bda9c4f
Update tfc-dump.pl
Praveenraj-K Jan 2, 2025
f2517d2
Update tfc-dump.pl
Praveenraj-K Jan 3, 2025
28d742e
Fix Variable Sets pagination to retrieve all pages
Praveenraj-K Jan 3, 2025
371cf24
Revise error messaging logic to work with multiple pages & Added Sentry
Praveenraj-K Jan 6, 2025
a8b8246
Adding cpanm for installing Sentry SDK
Praveenraj-K Jan 6, 2025
510bfda
Changed the Sentry SDK to Sentry CLI
Praveenraj-K Jan 23, 2025
d1b1c7a
Made changed for backward compatability, revised the error messaging …
Praveenraj-K Jan 24, 2025
89bfde6
Fixed the qoutes for consistency
Praveenraj-K Jan 24, 2025
141b9bd
Changing the Sentry installation method from npm to bash
Praveenraj-K Jan 27, 2025
cb7ee4a
Fixed the backslash and removed the bash from the dockerfile
Praveenraj-K Jan 27, 2025
7e16404
Added function slurp temp memory
Praveenraj-K Jan 28, 2025
e91f151
Replace manual temporary file management with file::temp and file::sl…
Praveenraj-K Feb 3, 2025
fa68649
Changed the page pagination to 100
Praveenraj-K Feb 5, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 9 additions & 2 deletions Dockerfile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,8 +5,9 @@ FROM alpine:3
ARG restic_ver=0.16.0
ARG tfc_ops_ver=3.5.4
ARG tfc_ops_distrib=tfc-ops_Linux_x86_64.tar.gz
ARG SENTRY_CLI_VERSION=2.41.1

# Install Restic, tfc-ops, perl, and jq
# Install Restic, tfc-ops, perl, jq, and sentry-cli
RUN cd /tmp \
&& wget -O /tmp/restic.bz2 \
https://github.com/restic/restic/releases/download/v${restic_ver}/restic_${restic_ver}_linux_amd64.bz2 \
Expand All @@ -18,7 +19,13 @@ RUN cd /tmp \
&& rm LICENSE README.md ${tfc_ops_distrib} \
&& mv tfc-ops /usr/local/bin \
&& apk update \
&& apk add --no-cache perl jq curl \
&& apk add --no-cache \
perl \
perl-file-slurp \
perl-file-temp \
jq \
curl \
&& curl -sL https://sentry.io/get-cli/ | SENTRY_CLI_VERSION="${SENTRY_CLI_VERSION}" sh \
&& rm -rf /var/cache/apk/*

COPY ./tfc-backup-b2.sh /usr/local/bin/tfc-backup-b2.sh
Expand Down
259 changes: 160 additions & 99 deletions tfc-dump.pl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,44 @@
use strict;
use warnings;
use Getopt::Long qw(GetOptions);
use File::Slurp qw(read_file);
mtompset marked this conversation as resolved.
Show resolved Hide resolved
use File::Temp;

# Function to log errors to Sentry
sub error_to_sentry {
my ($message) = @_;
system("sentry-cli send-event \"$message\" 2>/dev/null") if $ENV{SENTRY_DSN};
}

# Command output handling functions
sub slurp_to_file {
my ($command) = @_;
my $tmp = File::Temp->new();

system("$command > " . $tmp->filename);
if ($?) {
my $error = "Failed to execute command: $command";
error_to_sentry($error);
die "$error\n";
}
return $tmp;
}

sub slurp_string_output {
my ($command) = @_;
my $tmp = slurp_to_file($command);
my $content = read_file($tmp->filename);
chomp($content);
return $content;
}
mtompset marked this conversation as resolved.
Show resolved Hide resolved

sub slurp_array_output {
my ($command) = @_;
my $tmp = slurp_to_file($command);
my @content = read_file($tmp->filename);
chomp(@content);
return @content;
}

my $usage = "Usage: $0 --org=org-name {--workspace=name | --all} [--quiet] [--help]\n";
my $tfc_org_name; # Terraform Cloud organization name
Expand All @@ -30,140 +68,163 @@

Getopt::Long::Configure qw(gnu_getopt);
GetOptions(
'org|o=s' => \$tfc_org_name,
'workspace|w=s' => \$tfc_workspace_name,
'all|a' => \$all_workspaces,
'quiet|q' => \$quiet_mode,
'help|h' => \$help
'org|o=s' => \$tfc_org_name,
'workspace|w=s' => \$tfc_workspace_name,
'all|a' => \$all_workspaces,
'quiet|q' => \$quiet_mode,
'help|h' => \$help
) or die $usage;

die $usage if (!defined($tfc_org_name) || defined($help));
die $usage if ( defined($tfc_workspace_name) && defined($all_workspaces)); # can't have both
die $usage if (!defined($tfc_workspace_name) && !defined($all_workspaces)); # must have one

if (! $ENV{ATLAS_TOKEN}) {
print STDERR "Terraform Cloud access token must be in ATLAS_TOKEN environment variable.\n";
die $usage;
my $error = "Terraform Cloud access token must be in ATLAS_TOKEN environment variable.";
error_to_sentry($error);
print STDERR "$error\n";
die $usage;
}

my $curl_header1 = "--header \"Authorization: Bearer $ENV{ATLAS_TOKEN}\"";
my $curl_header2 = "--header \"Content-Type: application/vnd.api+json\"";
my $curl_headers = "$curl_header1 $curl_header2";
if (defined($quiet_mode)) {
$curl_headers .= " --no-progress-meter";
$curl_headers .= " --no-progress-meter";
}
my $curl_query;
my $curl_cmd;
my $jq_cmd;
my %workspace_list;
if (defined($tfc_workspace_name)) { # One workspace desired

# Get the workspace ID given the workspace name.

$curl_query = "\"https://app.terraform.io/api/v2/organizations/${tfc_org_name}/workspaces/${tfc_workspace_name}\"";
$curl_cmd = "curl $curl_headers $curl_query";
$jq_cmd = "jq '.data.id'";

$tfc_workspace_id = `$curl_cmd | $jq_cmd`;
$tfc_workspace_id =~ s/"//g;
chomp($tfc_workspace_id);

$workspace_list{$tfc_workspace_name} = $tfc_workspace_id;
if (defined($tfc_workspace_name)) { # One workspace desired
# Get the workspace ID given the workspace name.
$curl_query = "\"https://app.terraform.io/api/v2/organizations/${tfc_org_name}/workspaces/${tfc_workspace_name}\"";
$curl_cmd = "curl $curl_headers $curl_query";
$jq_cmd = "jq '.data.id'";

$tfc_workspace_id = slurp_string_output("$curl_cmd | $jq_cmd");
$tfc_workspace_id =~ s/"//g;

$workspace_list{$tfc_workspace_name} = $tfc_workspace_id;
}
else { # All workspaces desired
my $tfc_ops_cmd = "tfc-ops workspaces list --organization ${tfc_org_name} --attributes name,id";

my @result = `$tfc_ops_cmd`;

# tfc-ops prints two header lines before the data we want to see.
shift(@result); # remove "Getting list of workspaces ..."
shift(@result); # remove "name, id"
chomp(@result); # remove newlines

my $name;
my $id;
foreach (@result) {
($name, $id) = split(/, /, $_);
$workspace_list{$name} = $id;
}
my $tfc_ops_cmd = "tfc-ops workspaces list --organization ${tfc_org_name} --attributes name,id";
my @result = slurp_array_output($tfc_ops_cmd);
if ($?) {
my $error = "Failed to list workspaces";
error_to_sentry($error);
die "$error\n";
}

# tfc-ops prints two header lines before the data we want to see.
shift(@result); # remove "Getting list of workspaces ..."
shift(@result); # remove "name, id"

foreach (@result) {
my ($name, $id) = split(/, /, $_);
$workspace_list{$name} = $id;
}
}

# Dump the workspace and variable data to files.

foreach (sort keys %workspace_list) {

# Dump the workspace info
$curl_query = "\"https://app.terraform.io/api/v2/workspaces/$workspace_list{$_}\"";
$curl_cmd = "curl $curl_headers --output $_-attributes.json $curl_query";
system($curl_cmd);

# Dump the variables info
$curl_query = "\"https://app.terraform.io/api/v2/workspaces/$workspace_list{$_}/vars\"";
$curl_cmd = "curl $curl_headers --output $_-variables.json $curl_query";
system($curl_cmd);
# Dump the workspace info
$curl_query = "\"https://app.terraform.io/api/v2/workspaces/$workspace_list{$_}\"";
$curl_cmd = "curl $curl_headers --output $_-attributes.json $curl_query";
if (system($curl_cmd) != 0) {
my $error = "Failed to dump workspace attributes for $_";
error_to_sentry($error);
print STDERR "$error\n";
}

# Dump the variables info
$curl_query = "\"https://app.terraform.io/api/v2/workspaces/$workspace_list{$_}/vars\"";
$curl_cmd = "curl $curl_headers --output $_-variables.json $curl_query";
if (system($curl_cmd) != 0) {
my $error = "Failed to dump workspace variables for $_";
error_to_sentry($error);
print STDERR "$error\n";
}
}

# Dump the variable sets data to files.

### WARNING ###
#
# This code assumes that all of the TFC Variable Sets are contained within
# the first result page of 100 entries. This was true for SIL in August 2024.
#
####

my @vs_names;
my @vs_ids;
my $pg_size = 100;
my $pg_size = 100; # Default page size per Hashicorp documentation
my $pg_num = 1; # Start with page 1
my $total_count;
my $tmpfile = `mktemp`;
chomp($tmpfile);

$curl_query = "\"https://app.terraform.io/api/v2/organizations/${tfc_org_name}/varsets?page%5Bsize%5D=${pg_size}\"";
$curl_cmd = "curl $curl_headers --output $tmpfile $curl_query";
system($curl_cmd);

# Get the Variable Set names

$jq_cmd = "cat $tmpfile | jq '.data[].attributes.name'";
@vs_names = `$jq_cmd`;
# Remove the double quotes in each element of the array.
grep($_ =~ s/"//g && 0, @vs_names); # Programming Perl, p. 221, 1990
chomp(@vs_names);

# Get the Variable Set IDs
my $total_processed = 0;
my $tmp = File::Temp->new();

do {
# Get the current page of variable sets
$curl_query = "\"https://app.terraform.io/api/v2/organizations/${tfc_org_name}/varsets?page%5Bsize%5D=${pg_size}&page%5Bnumber%5D=${pg_num}\"";
$curl_cmd = "curl $curl_headers --output " . $tmp->filename . " $curl_query";

if (system($curl_cmd) != 0) {
my $error = "Failed to fetch variable sets page $pg_num";
error_to_sentry($error);
print STDERR "$error\n";
exit(1);
}

# Get the Variable Set names for this page
$jq_cmd = "cat " . $tmp->filename . " | jq '.data[].attributes.name'";
my @page_names = slurp_array_output($jq_cmd);
@page_names = map { s/"//gr } @page_names;
push(@vs_names, @page_names);

# Get the Variable Set IDs for this page
$jq_cmd = "cat " . $tmp->filename . " | jq '.data[].id'";
my @page_ids = slurp_array_output($jq_cmd);
@page_ids = map { s/"//gr } @page_ids;
push(@vs_ids, @page_ids);

# Get total count if we haven't yet
if (!defined($total_count)) {
$jq_cmd = "cat " . $tmp->filename . " | jq '.meta.pagination[\"total-count\"]'";
$total_count = slurp_string_output($jq_cmd);
print "Total variable sets to process: $total_count\n" unless defined($quiet_mode);
}

$total_processed += scalar(@page_names);
print "Processed $total_processed of $total_count variable sets\n" unless defined($quiet_mode);

$pg_num++;
} while ($total_processed < $total_count);

# Verify we got everything we expected
if ($total_processed != $total_count) {
my $error = "Warning: Expected $total_count variable sets but processed $total_processed";
error_to_sentry($error);
print STDERR "$error\n";
}

$jq_cmd = "cat $tmpfile | jq '.data[].id'";
@vs_ids = `$jq_cmd`;
# Remove the double quotes in each element of the array.
grep($_ =~ s/"//g && 0, @vs_ids); # Programming Perl, p. 221, 1990
chomp(@vs_ids);
print "Successfully processed all $total_count variable sets\n" unless defined($quiet_mode);

my $filename;
for (my $ii = 0; $ii < scalar @vs_names; $ii++) {
$filename = $vs_names[$ii];
$filename =~ s/ /-/g; # replace spaces with hyphens

# Get the Variable Set
$curl_query = "\"https://app.terraform.io/api/v2/varsets/$vs_ids[$ii]\"";
$curl_cmd = "curl $curl_headers --output varset-${filename}-attributes.json $curl_query";
system($curl_cmd);

# Get the variables within the Variable Set
$curl_query = "\"https://app.terraform.io/api/v2/varsets/$vs_ids[$ii]/relationships/vars\"";
$curl_cmd = "curl $curl_headers --output varset-${filename}-variables.json $curl_query";
system($curl_cmd);
}

# Get the number of Variable Sets

$jq_cmd = "cat $tmpfile | jq '.meta.pagination[\"total-count\"]'";
$total_count = `$jq_cmd`;
unlink($tmpfile);

if ($total_count > $pg_size) {
print STDERR "WARNING: ${total_count}-${pg_size} Variable Sets were not backed up.\n";
exit(1);
$filename = $vs_names[$ii];
$filename =~ s/ /-/g; # replace spaces with hyphens

# Get the Variable Set
$curl_query = "\"https://app.terraform.io/api/v2/varsets/$vs_ids[$ii]\"";
$curl_cmd = "curl $curl_headers --output varset-${filename}-attributes.json $curl_query";
if (system($curl_cmd) != 0) {
my $error = "Failed to dump varset attributes for $filename";
error_to_sentry($error);
print STDERR "$error\n";
}

# Get the variables within the Variable Set
$curl_query = "\"https://app.terraform.io/api/v2/varsets/$vs_ids[$ii]/relationships/vars\"";
$curl_cmd = "curl $curl_headers --output varset-${filename}-variables.json $curl_query";
if (system($curl_cmd) != 0) {
my $error = "Failed to dump varset variables for $filename";
error_to_sentry($error);
print STDERR "$error\n";
}
}

exit(0);