From 363efb9d19753668857a88e9745813e992219f42 Mon Sep 17 00:00:00 2001 From: Kazunari Sekigawa Date: Wed, 4 Oct 2023 22:08:11 +0900 Subject: [PATCH] Added Change_Python_LibPath_RelativeToAbsolute() function to force relative paths of libraries that Python depends on to absolute paths. Also added some code for debugging. --- macbuild/build4mac.py | 149 ++++++++++++++++++------ macbuild/build4mac_env.py | 21 ++++ macbuild/build4mac_util.py | 230 ++++++++++++++++++++++++++++++++++--- macbuild/findsharedlib.sh | 11 ++ 4 files changed, 357 insertions(+), 54 deletions(-) create mode 100755 macbuild/findsharedlib.sh diff --git a/macbuild/build4mac.py b/macbuild/build4mac.py index f72c2f180f..cf224a3e22 100755 --- a/macbuild/build4mac.py +++ b/macbuild/build4mac.py @@ -61,7 +61,7 @@ def GenerateUsage(platform): usage += " : Qt6Brew: use Qt6 from Homebrew (*) | \n" usage += " : (*) migration to Qt6 is ongoing | \n" usage += " [-r|--ruby ] : case-insensitive type=['nil', 'Sys', 'MP31', 'HB31', 'Ana3', | %s \n" % myRuby - usage += " : 'MP32', HB32'] | \n" + usage += " : 'MP32', 'HB32'] | \n" usage += " : nil: don't bind Ruby | \n" usage += " : Sys: use OS-bundled Ruby [2.0 - 2.6] depending on OS | \n" usage += " : MP31: use Ruby 3.1 from MacPorts | \n" @@ -70,7 +70,7 @@ def GenerateUsage(platform): usage += " : MP32: use Ruby 3.2 from MacPorts | \n" usage += " : HB32: use Ruby 3.2 from Homebrew | \n" usage += " [-p|--python ] : case-insensitive type=['nil', 'Sys', 'MP38', 'HB38', 'Ana3', | %s \n" % myPython - usage += " : 'MP39', HB39', 'HBAuto'] | \n" + usage += " : 'MP39', 'HB39', 'HBAuto'] | \n" usage += " : nil: don't bind Python | \n" usage += " : Sys: use OS-bundled Python 2.7 up to Catalina | \n" usage += " : MP38: use Python 3.8 from MacPorts | \n" @@ -154,6 +154,9 @@ def Get_Default_Config(): # Set the OS-wise usage and module set Usage, ModuleSet = GenerateUsage(Platform) + # developer's debug level list for this tool + ToolDebug = list() + # Set the default modules if Platform == "Ventura": ModuleQt = "Qt5Brew" @@ -226,6 +229,7 @@ def Get_Default_Config(): config['DeployVerbose'] = DeployVerbose # -verbose=<0-3> level passed to 'macdeployqt' tool config['Version'] = Version # KLayout's version config['ModuleSet'] = ModuleSet # (Qt, Ruby, Python)-tuple + config['ToolDebug'] = ToolDebug # debug level list for this tool # auxiliary variables on platform config['System'] = System # 6-tuple from platform.uname() config['Node'] = Node # - do - @@ -382,6 +386,7 @@ def Parse_CLI_Args(config): PackagePrefix = config['PackagePrefix'] DeployVerbose = config['DeployVerbose'] ModuleSet = config['ModuleSet'] + ToolDebug = config['ToolDebug'] #----------------------------------------------------- # [2] Parse the CLI arguments @@ -449,6 +454,11 @@ def Parse_CLI_Args(config): dest='deploy_verbose', help="verbose level of `macdeployqt` tool" ) + p.add_option( '-t', '--tooldebug', + action='append', + dest='tool_debug', + help="developer's debug level list for this tool" ) # not shown in the usage + p.add_option( '-?', '--??', action='store_true', dest='checkusage', @@ -468,6 +478,7 @@ def Parse_CLI_Args(config): deploy_full = False, deploy_partial = False, deploy_verbose = "1", + tool_debug = [], checkusage = False ) else: # with Xcode [ .. 12.4] p.set_defaults( type_qt = "qt5macports", @@ -482,6 +493,7 @@ def Parse_CLI_Args(config): deploy_full = False, deploy_partial = False, deploy_verbose = "1", + tool_debug = [], checkusage = False ) opt, args = p.parse_args() @@ -647,6 +659,7 @@ def Parse_CLI_Args(config): CheckComOnly = opt.check_command DeploymentF = opt.deploy_full DeploymentP = opt.deploy_partial + ToolDebug = sorted( set([ int(val) for val in opt.tool_debug ]) ) if DeploymentF and DeploymentP: print("") @@ -715,6 +728,7 @@ def Parse_CLI_Args(config): config['PackagePrefix'] = PackagePrefix config['DeployVerbose'] = DeployVerbose config['ModuleSet'] = ModuleSet + config['ToolDebug'] = ToolDebug if CheckComOnly: pp = pprint.PrettyPrinter( indent=4, width=140 ) @@ -1235,6 +1249,7 @@ def Deploy_Binaries_For_Bundle(config, parameters): DeployVerbose = config['DeployVerbose'] ModuleRuby = config['ModuleRuby'] ModulePython = config['ModulePython'] + ToolDebug = config['ToolDebug'] BuildPymod = parameters['BuildPymod'] ProjectDir = parameters['project_dir'] @@ -1521,10 +1536,9 @@ def Deploy_Binaries_For_Bundle(config, parameters): shutil.copy2( item, targetDirP ) print( " [7] Setting and changing the identification names of KLayout's libraries in each executable ..." ) - #------------------------------------------------------------- - # [7] Set and change the library identification name(s) of - # different executables - #------------------------------------------------------------- + #------------------------------------------------------------------------------------ + # [7] Set and change the library identification name(s) of different executable(s) + #------------------------------------------------------------------------------------ os.chdir(ProjectDir) os.chdir(MacPkgDir) klayoutexec = "klayout.app/Contents/MacOS/klayout" @@ -1551,14 +1565,14 @@ def Deploy_Binaries_For_Bundle(config, parameters): #------------------------------------------------------------- # [8] Deploy Qt Frameworks #------------------------------------------------------------- - verbose = " -verbose=%d" % DeployVerbose + verbose = " -verbose=%d" % DeployVerbose app_bundle = "klayout.app" - options = macdepQtOpt + verbose + options = macdepQtOpt + verbose deploytool = parameters['deploy_tool'] # Without the following, the plugin cocoa would not be found properly. shutil.copy2( sourceDir2 + "/qt.conf", targetDirM ) - os.chmod( targetDirM + "/qt.conf", 0o0644 ) + os.chmod( targetDirM + "/qt.conf", 0o0644 ) os.chdir(ProjectDir) os.chdir(MacPkgDir) @@ -1611,7 +1625,8 @@ def Deploy_Binaries_For_Bundle(config, parameters): deploymentPython39HB = (ModulePython == 'Python39Brew') deploymentPythonAutoHB = (ModulePython == 'PythonAutoBrew') if (deploymentPython38HB or deploymentPython39HB or deploymentPythonAutoHB) and NonOSStdLang: - from build4mac_util import WalkFrameworkPaths, PerformChanges + # from build4mac_util import WalkFrameworkPaths, PerformChanges + # from build4mac_util import Change_Python_LibPath_RelativeToAbsolute, DumpDependencyDic if deploymentPython38HB: HBPythonFrameworkPath = HBPython38FrameworkPath @@ -1619,6 +1634,7 @@ def Deploy_Binaries_For_Bundle(config, parameters): elif deploymentPython39HB: HBPythonFrameworkPath = HBPython39FrameworkPath pythonHBVer = "3.9" # 'pinned' to this version as of KLayout version 0.28.2 (2023-01-02) + # More specifically, "3.9.17" as of KLayout version 0.28.12 (2023-09-dd) elif deploymentPythonAutoHB: HBPythonFrameworkPath = HBPythonAutoFrameworkPath pythonHBVer = HBPythonAutoVersion @@ -1636,6 +1652,10 @@ def Deploy_Binaries_For_Bundle(config, parameters): print( "" ) print( " [9] Optional deployment of Python from %s ..." % HBPythonFrameworkPath ) print( " [9.1] Copying Python Framework" ) + if 910 in ToolDebug: + dbglevel = 910 + else: + dbglevel = 0 cmd01 = "rm -rf %s" % pythonFrameworkPath cmd02 = "rsync -a --safe-links %s/ %s" % (HBPythonFrameworkPath, pythonFrameworkPath) @@ -1660,7 +1680,7 @@ def Deploy_Binaries_For_Bundle(config, parameters): for command in shell_commands: if subprocess.call( command, shell=True ) != 0: - msg = "command failed: %s" + msg = "In [9.1], failed to execute command: %s" print( msg % command, file=sys.stderr ) sys.exit(1) @@ -1669,23 +1689,39 @@ def Deploy_Binaries_For_Bundle(config, parameters): os.chmod( targetDirM + "/start-console.py", 0o0755 ) os.chmod( targetDirM + "/klayout_console", 0o0755 ) - print( " [9.2] Relinking dylib dependencies inside Python.framework" ) + print( " [9.2] Re-linking dylib dependencies inside Python.framework" ) print( " [9.2.1] Patching Python Framework" ) - depdict = WalkFrameworkPaths( pythonFrameworkPath ) + if 921 in ToolDebug: + dbglevel = 921 + else: + dbglevel = 0 + Change_Python_LibPath_RelativeToAbsolute( pythonFrameworkPath, debug_level=dbglevel ) + depdict = WalkFrameworkPaths( pythonFrameworkPath, debug_level=dbglevel ) + DumpDependencyDic( "[9.2.1]", depdict, debug_level=dbglevel ) appPythonFrameworkPath = '@executable_path/../Frameworks/Python.framework/' - PerformChanges( depdict, [(HBPythonFrameworkPath, appPythonFrameworkPath, False)], bundleExecPathAbs ) + replacePairs = [ (HBPythonFrameworkPath, appPythonFrameworkPath, False) ] + PerformChanges( depdict, replacePairs, bundleExecPathAbs, debug_level=dbglevel ) print( " [9.2.2] Patching 'Python' itself in Python Framework" ) + if 922 in ToolDebug: + dbglevel = 922 + else: + dbglevel = 0 filterreg = r'\t+%s/(opt|Cellar)' % DefaultHomebrewRoot - Patch_Python_In_PythonFramework( pythonFrameworkPath, filter_regex=filterreg ) + Patch_Python_In_PythonFramework( pythonFrameworkPath, filter_regex=filterreg, debug_level=dbglevel ) print( " [9.2.3] Patching %s/opt/ libs" % DefaultHomebrewRoot ) # eg. DefaultHomebrewRoot == "/usr/local" + if 923 in ToolDebug: + dbglevel = 923 + else: + dbglevel = 0 + filterreg = r'\t+%s/(opt|Cellar)' % DefaultHomebrewRoot + depdict = WalkFrameworkPaths( pythonFrameworkPath, search_path_filter=filterreg, debug_level=dbglevel ) + DumpDependencyDic( "[9.2.3]", depdict, debug_level=dbglevel ) usrLocalPath = '%s/opt/' % DefaultHomebrewRoot appUsrLocalPath = '@executable_path/../Frameworks/' replacePairs = [ (usrLocalPath, appUsrLocalPath, True) ] - filterreg = r'\t+%s/(opt|Cellar)' % DefaultHomebrewRoot - depdict = WalkFrameworkPaths( pythonFrameworkPath, search_path_filter=filterreg ) - PerformChanges( depdict, replacePairs, bundleExecPathAbs ) + PerformChanges( depdict, replacePairs, bundleExecPathAbs, debug_level=dbglevel ) #--------------------------------------------------------------------------------------------------- # As of 2023-07-09 (KLayout 0.28.10), @@ -1709,8 +1745,21 @@ def Deploy_Binaries_For_Bundle(config, parameters): # sqlite # xz #--------------------------------------------------------------------------------------------------- + # https://formulae.brew.sh/formula/python@3.9#default + # as of 2023-09-22, python@3.9 depends on: + # gdbm 1.23 GNU database manager + # mpdecimal 2.5.1 Library for decimal floating point arithmetic + # openssl@3 3.1.2 Cryptography and SSL/TLS Toolkit + # readline 8.2.1 Library for command-line editing + # sqlite 3.43.1 Command-line interface for SQLite + # xz 5.4.4 General-purpose data compression with high compression ratio + #--------------------------------------------------------------------------------------------------- if Platform in ['Catalina']: - print( " [9.2.4] Patching openssl@1.1, gdbm, mpdecimal, readline, sqlite, xz" ) + print( " [9.2.4] Patching [openssl@1.1, gdbm, mpdecimal, readline, sqlite, xz]" ) + if 924 in ToolDebug: + dbglevel = 924 + else: + dbglevel = 0 usrLocalPath = '%s/opt/' % DefaultHomebrewRoot appUsrLocalPath = '@executable_path/../Frameworks/' replacePairs = [ (usrLocalPath, appUsrLocalPath, True) ] @@ -1722,9 +1771,15 @@ def Deploy_Binaries_For_Bundle(config, parameters): pythonFrameworkPath + '/../mpdecimal', pythonFrameworkPath + '/../readline', pythonFrameworkPath + '/../sqlite', - pythonFrameworkPath + '/../xz'], search_path_filter=filterreg ) + pythonFrameworkPath + '/../xz'], + search_path_filter=filterreg, + debug_level=dbglevel ) else: # [ 'Ventura', 'Monterey', 'BigSur' ] - print( " [9.2.4] Patching openssl@3, gdbm, mpdecimal, readline, sqlite, xz" ) + print( " [9.2.4] Patching [openssl@3, gdbm, mpdecimal, readline, sqlite, xz]" ) + if 924 in ToolDebug: + dbglevel = 924 + else: + dbglevel = 0 usrLocalPath = '%s/opt/' % DefaultHomebrewRoot appUsrLocalPath = '@executable_path/../Frameworks/' replacePairs = [ (usrLocalPath, appUsrLocalPath, True) ] @@ -1736,19 +1791,39 @@ def Deploy_Binaries_For_Bundle(config, parameters): pythonFrameworkPath + '/../mpdecimal', pythonFrameworkPath + '/../readline', pythonFrameworkPath + '/../sqlite', - pythonFrameworkPath + '/../xz'], search_path_filter=filterreg ) - PerformChanges( depdict, replacePairs, bundleExecPathAbs ) + pythonFrameworkPath + '/../xz'], + search_path_filter=filterreg, + debug_level=dbglevel ) - print( " [9.3] Relinking dylib dependencies for klayout" ) + DumpDependencyDic( "[9.2.4]", depdict, debug_level=dbglevel ) + PerformChanges( depdict, replacePairs, bundleExecPathAbs, debug_level=dbglevel ) + + print( " [9.3] Re-linking dylib dependencies for klayout" ) + if 931 in ToolDebug: + dbglevel = 931 + else: + dbglevel = 0 klayoutPath = bundleExecPathAbs - depdict = WalkFrameworkPaths( klayoutPath, filter_regex=r'klayout$' ) - PerformChanges( depdict, [(HBPythonFrameworkPath, appPythonFrameworkPath, False)], bundleExecPathAbs ) + depdict = WalkFrameworkPaths( klayoutPath, filter_regex=r'klayout$', debug_level=dbglevel ) + DumpDependencyDic( "[9.3.1]", depdict, debug_level=dbglevel ) + replacePairs = [ (HBPythonFrameworkPath, appPythonFrameworkPath, False) ] + PerformChanges( depdict, replacePairs, bundleExecPathAbs, debug_level=dbglevel ) + if 932 in ToolDebug: + dbglevel = 932 + else: + dbglevel = 0 libKlayoutPath = bundleExecPathAbs + '../Frameworks' - depdict = WalkFrameworkPaths( libKlayoutPath, filter_regex=r'libklayout' ) - PerformChanges( depdict, [(HBPythonFrameworkPath, appPythonFrameworkPath, False)], bundleExecPathAbs ) + depdict = WalkFrameworkPaths( libKlayoutPath, filter_regex=r'libklayout', debug_level=dbglevel ) + DumpDependencyDic( "[9.3.2]", depdict, debug_level=dbglevel ) + replacePairs = [ (HBPythonFrameworkPath, appPythonFrameworkPath, False) ] + PerformChanges( depdict, replacePairs, bundleExecPathAbs, debug_level=dbglevel ) print( " [9.4] Patching site.py, pip/, and distutils/" ) + if 940 in ToolDebug: + dbglevel = 940 + else: + dbglevel = 0 site_module = "%s/Versions/%s/lib/python%s/site.py" % (pythonFrameworkPath, pythonHBVer, pythonHBVer) with open(site_module, 'r') as site: buf = site.readlines() @@ -1778,10 +1853,10 @@ def Deploy_Binaries_For_Bundle(config, parameters): # Type "help", "copyright", "credits" or "license" for more information. # (KLayout Python Console) # >>> import pip - # >>> pip.main( ['install', 'numpy'] ) - # >>> pip.main( ['install', 'scipy'] ) - # >>> pip.main( ['install', 'pandas'] ) - # >>> pip.main( ['install', 'matplotlib'] ) + # >>> pip.main( ['install', 'pandas', 'scipy', 'matplotlib'] ) + # >>> quit() + # + # 'pandas' depends on many modules including 'numpy'. They are also installed. #---------------------------------------------------------------------------------- pip_module = "%s/Versions/%s/lib/python%s/site-packages/pip/__init__.py" % \ (pythonFrameworkPath, pythonHBVer, pythonHBVer) @@ -1807,15 +1882,15 @@ def Deploy_Binaries_For_Bundle(config, parameters): file.write(line) #------------------------------------------------------------- - # [10] Special deployment of Ruby3.1 from Homebrew? + # [10] Special deployment of Ruby3.2 from Homebrew? #------------------------------------------------------------- - deploymentRuby31HB = (ModuleRuby == 'Ruby31Brew') - if deploymentRuby31HB and NonOSStdLang: + deploymentRuby32HB = (ModuleRuby == 'Ruby32Brew') + if deploymentRuby32HB and NonOSStdLang: print( "" ) - print( " [10] You have reached optional deployment of Ruby from %s ..." % HBRuby31Path ) + print( " [10] You have reached optional deployment of Ruby from %s ..." % HBRuby32Path ) print( " [!!!] Sorry, the deployed package will not work properly since deployment of" ) - print( " Ruby2.7 from Homebrew is not yet supported." ) + print( " Ruby3.2 from Homebrew is not yet supported." ) print( " Since you have Homebrew development environment, there two options:" ) print( " (1) Retry to make a package with '-Y|--DEPLOY' option." ) print( " This will not deploy any of Qt[5|6], Python, and Ruby from Homebrew." ) diff --git a/macbuild/build4mac_env.py b/macbuild/build4mac_env.py index ef2e8ff0da..7eb0778913 100755 --- a/macbuild/build4mac_env.py +++ b/macbuild/build4mac_env.py @@ -27,8 +27,29 @@ (System, Node, Release, MacVersion, Machine, Processor) = platform.uname() if Machine == "arm64": # Apple Silicon! DefaultHomebrewRoot = '/opt/homebrew' + HomebrewSearchPathFilter1 = '\t+%s/opt' % DefaultHomebrewRoot + HomebrewSearchPathFilter2 = '\t+@loader_path/../../../../../../../../../../opt' + HomebrewSearchPathFilter3 = '@loader_path/../../../../../../../../../../opt' # no leading white space + # 1: absolute path as in ~python@3.9.17 + # 2: relative path as in python@3.9.18~ else: DefaultHomebrewRoot = '/usr/local' + HomebrewSearchPathFilter1 = '\t+%s/opt' % DefaultHomebrewRoot + HomebrewSearchPathFilter2 = '\t+@loader_path/../../../../../../../../../../opt' + HomebrewSearchPathFilter3 = '@loader_path/../../../../../../../../../../opt' # no leading white space + # 1: absolute path as in ~python@3.9.17 + # BigSur{kazzz-s} lib-dynload (1)% otool -L _sqlite3.cpython-39-darwin.so + # _sqlite3.cpython-39-darwin.so: + # ===> /usr/local/opt/sqlite/lib/libsqlite3.0.dylib (compatibility version 9.0.0, current version 9.6.0) + # /usr/lib/libSystem.B.dylib (compatibility version 1.0.0, current version 1292.100.5) + # + # 2: relative path as in python@3.9.18~ + # Monterey{kazzz-s} lib-dynload (1)% otool -L _sqlite3.cpython-39-darwin.so + # _sqlite3.cpython-39-darwin.so: + # ===> @loader_path/../../../../../../../../../../opt/sqlite/lib/libsqlite3.0.dylib (compatibility version 9.0.0, current version 9.6.0) + # /usr/lib/libSystem.B.dylib (compatibility version 1.0.0, current version 1311.100.3) + # + # Ref. https://github.com/Homebrew/homebrew-core/issues/140930#issuecomment-1701524467 del System, Node, Release, MacVersion, Machine, Processor #----------------------------------------------------- diff --git a/macbuild/build4mac_util.py b/macbuild/build4mac_util.py index 022805ffb3..0cb3969c85 100755 --- a/macbuild/build4mac_util.py +++ b/macbuild/build4mac_util.py @@ -184,28 +184,40 @@ def SetChangeLibIdentificationName( executable, relativedir ): #---------------------------------------------------------------------------------------- ## To make a library dependency dictionary by recursively walk down the lib hierarchy -# DefaultHomebrewRoot = '/opt/homebrew' "arm64" Apple Silicon -# DefaultHomebrewRoot = '/usr/local' "x86_64" Intel Mac +# Refer to "macbuild/build4mac_env.py" for HomebrewSearchPathFilter[1|2] # # @param[in] dylibPath: dylib path # @param[in] depth: hierarchy depth (< 5) # @param[in] filter_regex: filter regular expression +# @param[in] debug_level: debug level # # @return a dictionary #---------------------------------------------------------------------------------------- -def WalkLibDependencyTree( dylibPath, depth=0, filter_regex=r'\t+%s/opt' % DefaultHomebrewRoot ): +def WalkLibDependencyTree( dylibPath, + depth=0, + filter_regex=r'%s' % HomebrewSearchPathFilter1, + debug_level=0 ): + otoolCm = 'otool -L %s | grep -E "%s"' % (dylibPath, filter_regex) otoolOut = os.popen( otoolCm ).read() exedepdic = DecomposeLibraryDependency( dylibPath + ":\n" + otoolOut ) keys = exedepdic.keys() deplibs = exedepdic[ list(keys)[0] ] + if debug_level > 0: + print( "In WalkLibDependencyTree()" ) + print( " 1) depth = %d" % depth ) + print( " 2) dylibPath = %s" % dylibPath ) + print( " 3) exedepdic = %s" % exedepdic ) + print( " 4) key = %s" % list(keys)[0] ) + print( " 5) deplibs = %s" % deplibs ) + if depth < 5: if len(deplibs) > 0: for idx, lib in enumerate(deplibs): lib = str(lib) if lib != list(keys)[0]: - deplibs[idx] = WalkLibDependencyTree( lib, depth+1, filter_regex ) + deplibs[idx] = WalkLibDependencyTree( lib, depth+1, filter_regex, debug_level ) if depth == 0: return deplibs return exedepdic @@ -214,17 +226,20 @@ def WalkLibDependencyTree( dylibPath, depth=0, filter_regex=r'\t+%s/opt' % Defau #---------------------------------------------------------------------------------------- ## To make a library dependency dictionary by recursively walk down the Framework -# DefaultHomebrewRoot = '/opt/homebrew' "arm64" Apple Silicon -# DefaultHomebrewRoot = '/usr/local' "x86_64" Intel Mac +# Refer to "macbuild/build4mac_env.py" for HomebrewSearchPathFilter[1|2] # # @param[in] frameworkPaths: Framework path # @param[in] filter_regex: filter regular expression # @param[in] search_path_filter: search path filter regular expression +# @param[in] debug_level: debug level # # @return a dictionary #---------------------------------------------------------------------------------------- -def WalkFrameworkPaths( frameworkPaths, filter_regex=r'\.(so|dylib)$', - search_path_filter=r'\t+%s/opt' % DefaultHomebrewRoot ): +def WalkFrameworkPaths( frameworkPaths, + filter_regex=r'\.(so|dylib)$', + search_path_filter=r'%s' % HomebrewSearchPathFilter1, + debug_level=0 ): + if isinstance( frameworkPaths, str ): frameworkPathsIter = [frameworkPaths] else: @@ -239,12 +254,45 @@ def WalkFrameworkPaths( frameworkPaths, filter_regex=r'\.(so|dylib)$', dependency_dict[frameworkPath] = list() for idx, file in enumerate(framework_files): - dict_file = { file: WalkLibDependencyTree( file, filter_regex=search_path_filter ) } + dict_dep = WalkLibDependencyTree( file, filter_regex=search_path_filter, debug_level=debug_level ) + if debug_level > 0: + print( "" ) + print( "Return of WalkLibDependencyTree() for <%s>" % file ) + print( " *) %s" % dict_dep ) + print( "" ) + dict_file = { file: dict_dep } dependency_dict[frameworkPath].append(dict_file) return dependency_dict #---------------------------------------------------------------------------------------- -## To make a list of changed libraries +## To dump the contents of a dependency dictionary +# +# @param[in] title: title +# @param[in] depdic: dependency dictionary to dump +# @param[in] debug_level: debug level +# +# @return void +#---------------------------------------------------------------------------------------- +def DumpDependencyDic( title, depdic, debug_level=0 ): + if not debug_level > 0: + return + + print( "### Dependency Dictionary <%s> ###" % title ) + count1 = 0 + for key1 in sorted(depdic.keys()): + count1 += 1 + diclist = depdic[key1] + print( " %3d:%s" % (count1, key1) ) + + count2 = 0 + for dict_file in diclist: + for key2 in sorted(dict_file.keys()): + count2 += 1 + val2 = dict_file[key2] + print( " %3d:%s:%s" % (count2, key2, val2) ) + +#---------------------------------------------------------------------------------------- +## To make a list of libraries to change # # @param[in] dependencyDict: library dependency dictionary # @param[in] visited_files: list of visited files @@ -307,7 +355,7 @@ def ResolveExecutablePath( path, executable_path ): return p #---------------------------------------------------------------------------------------- -## To detect the changed library names +## To detect the library names to change # # @param[in] frameworkDependencyDict: framework dependency dictionary # @@ -330,12 +378,27 @@ def DetectChanges(frameworkDependencyDict): # @param[in] frameworkDependencyDict: framework dependency dictionary # @param[in] replaceFromToPairs: (from, to)-pair for replacement # @param[in] executable_path: executable path +# @param[in] debug_level: debug level # # @return 0 on success; > 0 on failure #---------------------------------------------------------------------------------------- -def PerformChanges( frameworkDependencyDict, replaceFromToPairs=None, executable_path="/tmp/klayout" ): +def PerformChanges( frameworkDependencyDict, + replaceFromToPairs=None, + executable_path="/tmp/klayout", + debug_level=0 ): + libNameChanges = DetectChanges(frameworkDependencyDict) - #print(libNameChanges) + # eg libNameChanges = [ ('lib.dylib',), ('lib.dylib',), ('lib.dylib', ['dep1.dylib', ...]), ... ] + if debug_level > 0: + print( "" ) + print( "PerformChanges() ---> DetectChanges()" ) + for tuple_item in libNameChanges: + if len(tuple_item) == 1: + print( " %s" % tuple_item[0] ) + elif len(tuple_item) == 2: + print( " %s, %s" % (tuple_item[0], tuple_item[1]) ) + print( "" ) + cmdNameId = XcodeToolChain['nameID'] cmdNameChg = XcodeToolChain['nameCH'] @@ -344,13 +407,18 @@ def PerformChanges( frameworkDependencyDict, replaceFromToPairs=None, executable else: for libNameChange in libNameChanges: libNameChangeIterator = iter(libNameChange) - lib = next(libNameChangeIterator) + lib = next(libNameChangeIterator) # 'lib.dylib' + if debug_level > 0: + print( "PerformChanges():lib = %s" % lib ) try: - dependencies = next(libNameChangeIterator) + dependencies = next(libNameChangeIterator) # dependencies = ['dep1.dylib', ...] if any except StopIteration: + # if libNameChange == ('lib.dylib',) dependencies = list() for replaceFrom, replaceTo, libdir in replaceFromToPairs: fileName = ResolveExecutablePath(lib.replace(replaceFrom, replaceTo), executable_path) + if debug_level > 0: + print( "PerformChanges():fileName = %s" % fileName ) if fileName.startswith('/usr'): # print(f'skipping fileName: {fileName}') continue @@ -368,7 +436,7 @@ def PerformChanges( frameworkDependencyDict, replaceFromToPairs=None, executable print( " COPYING:", frameworkPath, " -> ", destFrameworkPath ) shutil.copytree(frameworkPath, destFrameworkPath) - nameId = lib.replace(replaceFrom, replaceTo) + nameId = lib.replace(replaceFrom, replaceTo) command = "%s %s %s" % ( cmdNameId, nameId, fileName ) if not os.access(fileName, os.W_OK): command = "chmod u+w %s; %s; chmod u-w %s" % (fileName, command, fileName) @@ -511,10 +579,13 @@ def GenerateInfoPlist( keydic, templfile ): # # @param[in] pythonFrameworkPath: Python Framework path # @param[in] filter_regex: filter regular expression +# @param[in] debug_level: debug level # # @return 0 on succcess; non-zero on failure #---------------------------------------------------------------------------------------- -def Patch_Python_In_PythonFramework( pythonFrameworkPath, filter_regex=r'\t+%s/opt' % DefaultHomebrewRoot ): +def Patch_Python_In_PythonFramework( pythonFrameworkPath, + filter_regex=r'\t+%s/opt' % DefaultHomebrewRoot, + debug_level=0 ): #---------------------------------------------------------------------- # [1] Get Python's dependency #---------------------------------------------------------------------- @@ -549,6 +620,131 @@ def Patch_Python_In_PythonFramework( pythonFrameworkPath, filter_regex=r'\t+%s/o # for-lib return 0 +#---------------------------------------------------------------------------------------- +## To change the Python's relative library paths to the absolute paths +# +# 1: absolute path as in ~python@3.9.17 +# BigSur{kazzz-s} lib-dynload (1)% otool -L _sqlite3.cpython-39-darwin.so +# _sqlite3.cpython-39-darwin.so: +# ===> /usr/local/opt/sqlite/lib/libsqlite3.0.dylib (compatibility version 9.0.0, current version 9.6.0) +# /usr/lib/libSystem.B.dylib (compatibility version 1.0.0, current version 1292.100.5) +# +# 2: relative path as in python@3.9.18~ +# Monterey{kazzz-s} lib-dynload (1)% otool -L _sqlite3.cpython-39-darwin.so +# _sqlite3.cpython-39-darwin.so: +# ===> @loader_path/../../../../../../../../../../opt/sqlite/lib/libsqlite3.0.dylib (compatibility version 9.0.0, current version 9.6.0) +# /usr/lib/libSystem.B.dylib (compatibility version 1.0.0, current version 1311.100.3) +# +# @param[in] frameworkPath: Python Framework path +# @param[in] debug_level: debug level +# +# @return 0 on success; non-zero on failure +#---------------------------------------------------------------------------------------- +def Change_Python_LibPath_RelativeToAbsolute( frameworkPath, debug_level=0 ): + #---------------------------------------------------------------------- + # [1] Populate a dependency dictionary + #---------------------------------------------------------------------- + dependency_dict = dict() + filter_regex = r'\.(so|dylib)$' + patRel2 = r'(%s)(.+)' % HomebrewSearchPathFilter2 # = '\t+@loader_path/../../../../../../../../../../opt' + patRel3 = r'(%s)(.+)' % HomebrewSearchPathFilter3 # = '@loader_path/../../../../../../../../../../opt' + regRel3 = re.compile(patRel3) + + #---------------------------------------------------------------------- + # (A) Collect *.[so|dylib] that the Python Frameworks depends on + #---------------------------------------------------------------------- + # Ref. https://formulae.brew.sh/formula/python@3.9#default + # as of 2023-09-22, python@3.9 depends on: + # gdbm 1.23 GNU database manager + # mpdecimal 2.5.1 Library for decimal floating point arithmetic + # openssl@3 3.1.2 Cryptography and SSL/TLS Toolkit + # readline 8.2.1 Library for command-line editing + # sqlite 3.43.1 Command-line interface for SQLite + # xz 5.4.4 General-purpose data compression with high compression ratio + find_grep_results = os.popen( 'find %s -type f | grep -E "%s"' % (frameworkPath, filter_regex) ).read().split('\n') + framework_files = filter( lambda x: x != '', map(lambda x: x.strip(), find_grep_results) ) + + for idx, dylibPath in enumerate(framework_files): + otoolCm = 'otool -L %s | grep -E "%s"' % (dylibPath, patRel2) + otoolOut = os.popen( otoolCm ).read() + libdepdic = DecomposeLibraryDependency( dylibPath + ":\n" + otoolOut ) + keys = libdepdic.keys() + deplibs = libdepdic[ list(keys)[0] ] + + if len(deplibs) == 0: + continue + + if debug_level > 0: + print( "In Change_Python_LibPath_RelativeToAbsolute()" ) + print( " 1) dylibPath = %s" % dylibPath ) + print( " 2) libdepdic = %s" % libdepdic ) + print( " 3) key = %s" % list(keys)[0] ) + print( " 4) deplibs = %s" % deplibs ) + + # @LOADER_PATH = @loader_path/../../../../../../../../../.. + # dylibPath = /Abs/python3.9/lib-dynload/_hashlib.cpython-39-darwin.so + # libdepdic = {'/Abs/python3.9/lib-dynload/_hashlib.cpython-39-darwin.so': + # ['@LOADER_PATH/opt/openssl@3/lib/libssl.3.dylib', + # '@LOADER_PATH/opt/openssl@3/lib/libcrypto.3.dylib']} + # key = /Abs/python3.9/lib-dynload/_hashlib.cpython-39-darwin.so + # deplibs = ['@LOADER_PATH/opt/openssl@3/lib/libssl.3.dylib', + # '@LOADER_PATH/opt/openssl@3/lib/libcrypto.3.dylib'] + + for key in keys: + for file in libdepdic[key]: + if regRel3.match(file): + g1, g2 = regRel3.match(file).groups() + try: + container = dependency_dict[key] + except KeyError: + dependency_dict[key] = list() # new empty container + else: + pass + pathRel = "%s" % file + pathAbs = ("%s/opt" % DefaultHomebrewRoot) + g2 + dependency_dict[key].append( {pathRel:pathAbs} ) + + if len(dependency_dict) == 0: + print( " ---> Change_Python_LibPath_RelativeToAbsolute(): No need to change the library paths." ) + return 0 + + if debug_level > 0: + print( "In [1] of Change_Python_LibPath_RelativeToAbsolute()" ) + for key in sorted(dependency_dict.keys()): + val = dependency_dict[key] + print( " key=%s" % key ) + print( " val=%s" % val ) + + #---------------------------------------------------------------------- + # [2] Perform the changes: relative paths ---> absolute paths + #---------------------------------------------------------------------- + cmdNameId = XcodeToolChain['nameID'] + cmdNameChg = XcodeToolChain['nameCH'] + + if debug_level > 0: + print( "In [2] of Change_Python_LibPath_RelativeToAbsolute()" ) + + for targetfile in sorted(dependency_dict.keys()): + for depdic in dependency_dict[targetfile]: + nameOld = list(depdic.keys())[0] # relative path + nameNew = depdic[nameOld] # absolute path + + #----------------------------------------------------------- + # (A) Make the library aware of the new identification + # $ install_name_tool [-change old new] input + #----------------------------------------------------------- + command = "%s %s %s %s" % ( cmdNameChg, nameOld, nameNew, targetfile ) + if debug_level > 0: + print( " executing: %s" % command ) + if subprocess.call( command, shell=True ) != 0: + msg = "!!! Failed to make the library <%s> aware of the new identification name <%s> !!!" + print( msg % (targetfile, nameNew), file=sys.stderr ) + return 1 + # for-targetfile + + print( " ---> Change_Python_LibPath_RelativeToAbsolute(): Changed the library paths." ) + return 0 + #---------------- # End of File #---------------- diff --git a/macbuild/findsharedlib.sh b/macbuild/findsharedlib.sh new file mode 100755 index 0000000000..f0af990b47 --- /dev/null +++ b/macbuild/findsharedlib.sh @@ -0,0 +1,11 @@ +#!/bin/bash + +if [ $# -gt 0 ]; then + find $1 -type f | grep -E "\.(so|dylib)$" +else + echo "### Usage of 'findsharelib.sh' ###" + echo " To find shared libraries *.[so|dylib] under a " + echo "" + echo " $ findsharelib.sh " + echo "" +fi