From 08df9d50a4977029e29b22b982526326e9aaac28 Mon Sep 17 00:00:00 2001
From: Phil Rzewski <phil@brimdata.io>
Date: Fri, 24 Mar 2023 21:41:41 +0000
Subject: [PATCH] Fixes for recent issues (#10)

---
 README.md  | 6 ++++++
 migrate.sh | 9 ++++++++-
 2 files changed, 14 insertions(+), 1 deletion(-)

diff --git a/README.md b/README.md
index b0e73e7..f025c0d 100644
--- a/README.md
+++ b/README.md
@@ -26,6 +26,12 @@ of each pool is migrated.
 [time travel](https://zed.brimdata.io/docs/commands/zed#15-time-travel)
 will not be possible to pre-migration commits.
 
+3. If you have a very large number of pools to migrate, the script may fail
+with a `too many open files` error. This can be addressed by increasing the
+system's hard limit on the number of file descriptors. Many references are
+available that advise how this may be done, such as
+[this article](https://docs.oracle.com/cd/E93962_01/bigData.Doc/install_onPrem/src/tins_postinstall_file_descriptors.html).
+
 We expect most users are not yet dependent on the features affected by these
 limitations. However, if your environment is severely impacted by these
 limitations, come talk to us on the
diff --git a/migrate.sh b/migrate.sh
index 9281fba..fde7b0d 100755
--- a/migrate.sh
+++ b/migrate.sh
@@ -64,7 +64,7 @@ for pool_ksuid in $ksuid_glob; do
     pool_order=$(zq -f text "entry.id==ksuid('$pool_ksuid') | head 1 | yield join(entry.layout.keys[0], '.') + ':' + entry.layout.order" $pools_zngs)
 
     # Look for [0-9]*.zng so snap.zng is excluded
-    branch_count=$(zq -f text 'yield entry.name | sort | uniq | count()' $pool_ksuid/branches/[0-9]*.zng)
+    branch_count=$(zq -z 'yield entry.name | sort' $pool_ksuid/branches/[0-9]*.zng | zq -f text 'uniq | count()' -)
     if [ "$branch_count" != 1 ]; then
         echo "warning: found $branch_count branches in '$pool_name' ($pool_ksuid) but only migrating 'main'"
     fi
@@ -78,6 +78,13 @@ for pool_ksuid in $ksuid_glob; do
     prior_zng=$(mktemp)
     $prior_zed -lake "$src_dir" -use "$pool_name"@main query '*' > "$prior_zng"
     zed -lake "$dst_dir" create -q -orderby "$pool_order" "$pool_name"
+
+    if ! [ -s "$prior_zng" ]; then
+        echo "$pool_name is empty: no data to migrate"
+        rm -f "$prior_zng"
+        continue
+    fi
+
     zed -lake "$dst_dir" load -q -use "$pool_name" "$prior_zng"
     new_zng=$(mktemp)
     zed -lake "$dst_dir" -use "$pool_name" query '*' > "$new_zng"