Skip to content

Commit

Permalink
update one product build package logic (#423)
Browse files Browse the repository at this point in the history
  • Loading branch information
cynthiajoan authored Jul 28, 2022
1 parent 8fba07a commit a2b9bb5
Show file tree
Hide file tree
Showing 15 changed files with 4,599 additions and 27 deletions.
26 changes: 25 additions & 1 deletion .github/workflows/package.yml
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,7 @@ on:
env:
# Use SHA256 for hashing files.
hashCommand: "sha256sum"
default_apis: 'analytics,auth,crashlytics,database,dynamic_links,firestore,functions,installations,messaging,remote_config,storage'

jobs:
package_sdks:
Expand All @@ -49,6 +50,23 @@ jobs:
strategy:
fail-fast: false
steps:
- name: Check input
# This check the apis input. It detect whether the package is only asking for subset of
# of the products, like auth,firestore only etc. If the input apis are less than the
# default_apis env attribute, we package only the provided apis, for test purposes only.
id: check-input
shell: bash
run: |
IFS=',' read -r -a input_apis <<< "${{ inputs.apis }}"
IFS=',' read -r -a default_apis <<< "${{ env.default_apis }}"
if [[ ${#input_apis[@]} != ${#default_apis[@]} ]]; then
echo "::set-output name=package_for_checks::1"
echo "::set-output name=package_apis::'--apis=${{ inputs.apis }}'"
else
echo "::set-output name=package_for_checks::0"
echo "::set-output name=package_apis::"
fi
- name: Print inputs
shell: bash
run: |
Expand All @@ -63,6 +81,8 @@ jobs:
echo download_windows_run: ${{ inputs.download_windows_run }}
echo platforms: ${{ inputs.platforms }}"
echo apis: ${{ inputs.apis }}"
echo is the package for checks: ${{ steps.check-input.outputs.package_for_checks }}
echo package_apis: ${{ steps.check-input.outputs.package_apis }}
- name: Check out base branch
uses: actions/[email protected]
Expand Down Expand Up @@ -135,7 +155,7 @@ jobs:
- name: Package unitypackage
run: |
python scripts/build_scripts/build_package.py --zip_dir=built_artifact
python scripts/build_scripts/build_package.py --zip_dir=built_artifact ${{ steps.check-input.outputs.package_apis }}
- name: Commit Changes if there is any
if: inputs.working_branch != '' || inputs.create_new_branch == '1'
Expand Down Expand Up @@ -210,15 +230,18 @@ jobs:
path: firebase_unity_sdk.zip

- name: Package tgz
if: ${{ steps.check-input.outputs.package_for_checks }} == '1'
run: |
python scripts/build_scripts/build_package.py --zip_dir=built_artifact --output_upm=True --output=output_tgz
- name: Listing output tgz
if: ${{ steps.check-input.outputs.package_for_checks }} == '1'
run: |
ls -Rl
working-directory: output_tgz

- name: compute SDK hash for tgz files
if: ${{ steps.check-input.outputs.package_for_checks }} == '1'
shell: bash
run: |
tgz_files_list=$(find -type f -name '*.tgz')
Expand All @@ -229,6 +252,7 @@ jobs:
working-directory: output_tgz

- name: Upload Build tgz
if: ${{ steps.check-input.outputs.package_for_checks }} == '1'
uses: actions/upload-artifact@v2
with:
name: firebase_unity_sdk_tgz
Expand Down
105 changes: 80 additions & 25 deletions scripts/build_scripts/build_package.py
Original file line number Diff line number Diff line change
Expand Up @@ -39,12 +39,18 @@
from absl import flags
from absl import logging


SUPPORT_TARGETS = [
"analytics", "auth", "crashlytics", "database", "dynamic_links",
"firestore", "functions", "installations", "messaging", "remote_config",
"storage"
]

FLAGS = flags.FLAGS
flags.DEFINE_string('zip_dir', None,
'Directory of zip files from build output to package')
flags.DEFINE_string("config_file", "exports.json",
("Config file that describes how to "
"pack the unity assets."))
("Config file that describes how to pack the unity assets."))
flags.DEFINE_string("guids_file", "guids.json",
"Json file with stable guids cache.")

Expand All @@ -55,7 +61,13 @@
"Output folder for unitypackage.")

flags.DEFINE_boolean("output_upm", False, "Whether output packages as tgz for"
"Unity Package Manager.")
"Unity Package Manager.")

flags.DEFINE_string("apis", None, "which firebase products to pack, if not"
"set means package all. Value should be items in [{}],"
"connected with ',', eg 'auth,firestore'".format(
",".join(SUPPORT_TARGETS)))


def get_zip_files():
"""Get all zip files from FLAGS.zip_dir.
Expand All @@ -75,12 +87,14 @@ def get_zip_files():

return zip_file_paths


def get_last_version():
"""Get the last version number in guids file if exists.
Returns:
version number.
"""
version_cmake_path = os.path.join(os.getcwd(), "cmake", "firebase_unity_version.cmake")
version_cmake_path = os.path.join(
os.getcwd(), "cmake", "firebase_unity_version.cmake")
with open(version_cmake_path, "r") as f:
datafile = f.readlines()
for line in datafile:
Expand All @@ -89,45 +103,68 @@ def get_last_version():
return result[-1].strip("\"")
return None


def find_pack_script():
"""Get the pack script either from intermediate build folder or download from unity-jar-resolver.
Returns:
path of the pack script. None if not found.
"""
built_folder_ext = "_unity"
built_folder_postion = os.path.join("external", "src", "google_unity_jar_resolver")
built_folder_postion = os.path.join(
"external", "src", "google_unity_jar_resolver")
built_folder = None
resolver_root_folder = "unity-jar-resolver"
for folder in os.listdir("."):
if folder.endswith(built_folder_ext):
built_folder = folder
break

if built_folder != None:
resolver_root_folder = os.path.join(built_folder, built_folder_postion)
elif not os.path.exists(resolver_root_folder):
git_clone_script = ["git", "clone",
"--depth", "1",
"https://github.com/googlesamples/unity-jar-resolver.git"]
"--depth", "1",
"https://github.com/googlesamples/unity-jar-resolver.git"]
subprocess.call(git_clone_script)

if resolver_root_folder != None:
script_path = os.path.join(resolver_root_folder, "source", "ExportUnityPackage", "export_unity_package.py")
script_path = os.path.join(
resolver_root_folder, "source", "ExportUnityPackage", "export_unity_package.py")
return script_path
return None


def _debug_create_target_package(target, packer_script_path, guids_file_path, output_folder, zip_file_list, last_version):
debug_config_path = os.path.join(os.getcwd(), FLAGS.script_folder, "debug_single_export_json",
target+".json")
debug_cmd_args = [
sys.executable,
packer_script_path,
"--assets_dir=" + FLAGS.zip_dir,
"--config_file=" + debug_config_path,
"--guids_file=" + guids_file_path,
"--output_dir=" + output_folder,
"--output_upm=False",
]
debug_cmd_args.extend(
["--assets_zip=" + zip_file for zip_file in zip_file_list])
debug_cmd_args.append("--enabled_sections=build_dotnet4 asset_package_only")
debug_cmd_args.append("--plugins_version=" + last_version)
subprocess.call(debug_cmd_args)
logging.info("Debug Packaging done for target %s", target)


def main(argv):
if len(argv) > 1:
raise app.UsageError('Too many command-line arguments.')

packer_script_path = find_pack_script()
if packer_script_path == None:
raise app.UsageError('Cannot find pack script. Please build the project first.')

raise app.UsageError(
'Cannot find pack script. Please build the project first.')

packer_script_path = os.path.join(os.getcwd(), packer_script_path)
config_file_path = os.path.join(os.getcwd(), FLAGS.script_folder,
FLAGS.config_file)
guids_file_path = os.path.join(os.getcwd(), FLAGS.script_folder,
FLAGS.guids_file)

Expand All @@ -141,6 +178,21 @@ def main(argv):
if os.path.exists(output_folder):
shutil.rmtree(output_folder)

if FLAGS.apis:
# If told to only build a subset, package just those products and exit early.
api_list = FLAGS.apis.split(",")
if not set(api_list).issubset(set(SUPPORT_TARGETS)):
raise app.UsageError("apis parameter error, Value should be items in [{}],"
"connected with ',', eg 'auth,firestore'".format(
",".join(SUPPORT_TARGETS)))

for target in api_list:
_debug_create_target_package(
target, packer_script_path, guids_file_path, output_folder, zip_file_list, last_version)
return

config_file_path = os.path.join(os.getcwd(), FLAGS.script_folder,
FLAGS.config_file)
cmd_args = [
sys.executable,
packer_script_path,
Expand All @@ -151,16 +203,17 @@ def main(argv):
"--output_upm=" + str(FLAGS.output_upm),
]
cmd_args.extend(["--assets_zip=" + zip_file for zip_file in zip_file_list])

if last_version:
cmd_args.append("--plugins_version=" + last_version)

if FLAGS.output_upm:
cmd_args.append("--enabled_sections=build_dotnet4")
cmd_args.append("--output_unitypackage=False")
else:
cmd_args.append("--enabled_sections=build_dotnet3 build_dotnet4 asset_package_only")

cmd_args.append(
"--enabled_sections=build_dotnet3 build_dotnet4 asset_package_only")

# Check if need to gen new guids
p = subprocess.Popen(cmd_args, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
output, error = p.communicate()
Expand All @@ -170,28 +223,30 @@ def main(argv):
error_str = error_str.split("assets:")[-1]
error_str = error_str.rstrip("\\n\'")
split_string = error_str.split(" ")
split_string = split_string[3:] # exclude first 3 lines
gen_guids_script_path = os.path.join(os.getcwd(), "scripts", "build_scripts", "gen_guids.py")
split_string = split_string[3:] # exclude first 3 lines
gen_guids_script_path = os.path.join(
os.getcwd(), "scripts", "build_scripts", "gen_guids.py")
gen_cmd_args = [
sys.executable,
gen_guids_script_path,
"--guids_file=" + guids_file_path,
"--version=" + last_version,
"--generate_new_guids=True",
sys.executable,
gen_guids_script_path,
"--guids_file=" + guids_file_path,
"--version=" + last_version,
"--generate_new_guids=True",
]
for file in split_string:
file=file.strip("\"")
file = file.strip("\"")
print(file)
gen_cmd_args.append(file)
subprocess.call(gen_cmd_args)

# Need to package again if has that error
subprocess.call(cmd_args)
subprocess.call(cmd_args)
else:
logging.info("No new guid generated.")
error_list = str(error).split("\\n")
logging.info("\n".join(error_list))
logging.info("Packaging done for version %s", last_version)


if __name__ == '__main__':
app.run(main)
94 changes: 94 additions & 0 deletions scripts/create_debug_export.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,94 @@
#!/usr/bin/python
#
# Copyright 2022 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Based on the unity_packer/exports.json file, generate separate debug
export config for each product.
Example usage:
python scripts/build_scripts/create_debug_export.py
"""

import os
import json

from absl import app
from absl import flags
from absl import logging

API_PACKAGE_MAP = {
"analytics": "FirebaseAnalytics.unitypackage",
"auth": "FirebaseAuth.unitypackage",
"crashlytics": "FirebaseCrashlytics.unitypackage",
"database": "FirebaseDatabase.unitypackage",
"dynamic_links": "FirebaseDynamicLinks.unitypackage",
"firestore": "FirebaseFirestore.unitypackage",
"functions": "FirebaseFunctions.unitypackage",
"installations": "FirebaseInstallations.unitypackage",
"messaging": "FirebaseMessaging.unitypackage",
"remote_config": "FirebaseRemoteConfig.unitypackage",
"storage": "FirebaseStorage.unitypackage",
}

default_package_names = [
"FirebaseApp.unitypackage", "SampleCommon.unitypackage"]


FLAGS = flags.FLAGS
flags.DEFINE_string('prod_export', "exports.json",
'Production export json template to copy from')
flags.DEFINE_string("json_folder", "unity_packer",
("The root folder for json configs."))
flags.DEFINE_string("output_folder", "debug_single_export_json",
"Json file with stable guids cache.")


def main(argv):
if len(argv) > 1:
raise app.UsageError('Too many command-line arguments.')
prod_export_json_path = os.path.join(
os.getcwd(), FLAGS.json_folder, FLAGS.prod_export)

with open(prod_export_json_path, "r") as fin:
export_json = json.load(fin)

# making sure only dotnet4 exists
builds = export_json["builds"]
for idx, dict in enumerate(builds):
if dict["name"] == "dotnet3":
builds.pop(idx)

packages = export_json["packages"]

for api_name, package_name in API_PACKAGE_MAP.items():
output_path = os.path.join(
os.getcwd(), FLAGS.json_folder, FLAGS.output_folder, api_name + ".json")
output_dict = {}
output_package_list = []
for idx, package_dict in enumerate(packages):
if package_dict["name"] in default_package_names:
output_package_list.append(packages[idx])
elif package_dict["name"] == package_name:
output_package_list.append(packages[idx])
output_dict["packages"] = output_package_list
output_dict["builds"] = builds

with open(output_path, 'w', encoding='utf-8') as fout:
fout.write(json.dumps(output_dict, indent=2))
logging.info("Write %s", output_path)


if __name__ == '__main__':
app.run(main)
Loading

0 comments on commit a2b9bb5

Please sign in to comment.