PydMain; build improvements (no more colliding object files); example…

…s and docs updated to use PydMain

git-svn-id: http://svn.dsource.org/projects/pyd/trunk@64 1df65b71-e716-0410-9316-ac55df2b1602

dcompiler.py

@@ -141,7 +141,20 @@ def compile(self, sources,
         outputOpts = self._outputOpts
 
         includePathOpts = []
-
+
+        # All object files will be placed in one of three directories:
+        # infra   - All of the infrastructure's object files.
+        # project - The project's own object files.
+        # outside - Any source files specified by the project which are not
+        #           contained in the project's own directory.
+        orig_sources = sources
+        sources = []
+        for source in orig_sources:
+            if os.path.abspath(source).startswith(os.getcwd()):
+                sources.append((source, 'project'))
+            else:
+                sources.append((source, 'outside'))
+
         # To sources, add the appropriate D header file python.d, as well as
         # any platform-specific boilerplate.
         pythonHeaderPath = os.path.join(_infraDir, 'python', _pyVerXDotY, 'python.d')
@@ -152,44 +165,63 @@ def compile(self, sources,
             raise DistutilsPlatformError('Required D translation of Python'
                 ' header files "%s" is missing.' % pythonHeaderPath
             )
-        sources.append(pythonHeaderPath)
+        sources.append((pythonHeaderPath, 'infra'))
 
-        # flags = (with_pyd, with_st, with_meta)
-        flags = [f for f, category in macros if category == 'aux'][0]
+        # flags = (with_pyd, with_st, with_meta, with_main)
+        with_pyd, with_st, with_meta, with_main = [f for f, category in macros if category == 'aux'][0]
         # And Pyd!
-        if flags[0]:
+        if with_pyd:
             # If we're not using StackThreads, don't use iteration.d in Pyd
-            if not flags[1] or not self._st_support:
+            if not with_st or not self._st_support:
                 _pydFiles.remove('iteration.d');
             for file in _pydFiles:
                 filePath = os.path.join(_infraDir, 'pyd', file)
                 if not os.path.isfile(filePath):
                     raise DistutilsPlatformError("Required Pyd source file '%s' is"
                         " missing." % filePath
                     )
-                sources.append(filePath)
+                sources.append((filePath, 'infra'))
+        # If using PydMain, parse the template file
+        if with_main:
+            name = [n for n, category in macros if category == 'name'][0]
+            mainTemplatePath = os.path.join(_infraDir, 'd', 'pydmain_template.d')
+            if not os.path.isfile(mainTemplatePath):
+                raise DistutilsPlatformError(
+                    "Required supporting code file %s is missing." % mainTemplatePath
+                )
+            mainTemplate = open(mainTemplatePath).read()
+            mainFileContent = mainTemplate % {'modulename' : name}
+            # Store the finished pydmain.d file alongside the object files
+            infra_output_dir = os.path.join(output_dir, 'infra')
+            if not os.path.exists(infra_output_dir):
+                os.path.makedirs(infra_output_dir)
+            mainFilename = os.path.join(infra_output_dir, 'pydmain.d')
+            mainFile = open(mainFilename, 'w')
+            mainFile.write(mainFileContent)
+            mainFile.close()
+            sources.append((mainFilename, 'infra'))
         # And StackThreads
-        if self._st_support and flags[1]:
+        if self._st_support and with_st:
             for file in _stFiles:
                 filePath = os.path.join(_infraDir, 'st', file)
                 if not os.path.isfile(filePath):
                     raise DistutilsPlatformError("Required StackThreads source"
                         " file '%s' is missing." % filePath
                     )
-                sources.append(filePath)
+                sources.append((filePath, 'infra'))
             # Add the version conditional for st
             macros.append(('Pyd_with_StackThreads', 'version'))
         # And meta
-        if flags[2]:
+        if with_meta:
             for file in _metaFiles:
                 filePath = os.path.join(_infraDir, 'meta', file)
                 if not os.path.isfile(filePath):
                     raise DistutilsPlatformError("Required meta source file"
                         " '%s' is missing." % filePath
                     )
-                sources.append(filePath)
+                sources.append((filePath, 'infra'))
         # Add the infraDir to the include path for pyd, st, and meta.
-        if True in flags:
+        if True in (with_pyd, with_st, with_meta):
             includePathOpts += self._includeOpts
             includePathOpts[-1] = includePathOpts[-1] % os.path.join(_infraDir)
 
@@ -206,7 +238,7 @@ def compile(self, sources,
             raise DistutilsFileError('Required supporting code file "%s"'
                 ' is missing.' % boilerplatePath
             )
-        sources.append(boilerplatePath)
+        sources.append((boilerplatePath, 'infra'))
 
         # Extension subclass DExtension will have packed any user-supplied
         # version and debug flags into macros; we extract them and convert them
@@ -238,49 +270,34 @@ def compile(self, sources,
         else:
             optimizationOpts = self._defaultOptimizeOpts
 
-        print 'sources: ', [os.path.basename(s) for s in sources]
-        # Compiling one-by-one exhibits a strange bug in the D front-end, while
-        # compiling all at once works. This flags allows me to test each form
-        # easily. Supporting the one-by-one form is synonymous with GDC support.
-        ONE_BY_ONE = True
-        if ONE_BY_ONE:
-            for source in sources:
-                outOpts = outputOpts[:]
-                objName = self.object_filenames([os.path.split(source)[1]], 0, output_dir)[0]
-                outOpts[-1] = outOpts[-1] % _qp(objName)
-                cmdElements = (
-                    [binpath] + extra_preargs + compileOpts +
-                    [pythonVersionOpt, self._unicodeOpt] + optimizationOpts +
-                    includePathOpts + outOpts + userVersionAndDebugOpts +
-                    [_qp(source)] + extra_postargs
-                )
-                cmdElements = [el for el in cmdElements if el]
-                try:
-                    self.spawn(cmdElements)
-                except DistutilsExecError, msg:
-                    raise CompileError(msg)
-        else:
-            # gdc/gcc doesn't support the idea of an output directory, so we
-            # compile from the destination
-            sources = [_qp(os.path.abspath(s)) for s in sources]
-            cwd = os.getcwd()
-            os.chdir(output_dir)
+        print 'sources: ', [os.path.basename(s) for s, t in sources]
+
+        objFiles = []
+        for source, source_type in sources:
+            outOpts = outputOpts[:]
+            objFilename = os.path.splitext(source)[0] + self.obj_extension
+            if source_type == 'project':
+                objName = os.path.join(output_dir, 'project', objFilename)
+            elif source_type == 'outside':
+                objName = os.path.join(output_dir, 'outside', os.path.basename(objFilename))
+            else: # infra
+                objName = os.path.join(output_dir, 'infra', os.path.basename(objFilename))
+            if not os.path.exists(os.path.dirname(objName)):
+                os.makedirs(os.path.dirname(objName))
+            objFiles.append(objName)
+            outOpts[-1] = outOpts[-1] % _qp(objName)
             cmdElements = (
                 [binpath] + extra_preargs + compileOpts +
                 [pythonVersionOpt, self._unicodeOpt] + optimizationOpts +
-                includePathOpts + userVersionAndDebugOpts +
-                sources + extra_postargs
+                includePathOpts + outOpts + userVersionAndDebugOpts +
+                [_qp(source)] + extra_postargs
             )
             cmdElements = [el for el in cmdElements if el]
-
             try:
                 self.spawn(cmdElements)
             except DistutilsExecError, msg:
-                #os.chdir(cwd)
                 raise CompileError(msg)
-            os.chdir(cwd)
-
-        return [os.path.join(output_dir, fn) for fn in os.listdir(output_dir) if fn.endswith(self.obj_extension)]
+        return objFiles
 
     def link (self,
         target_desc, objects, output_filename,
@@ -431,7 +448,7 @@ def _def_file(self, output_dir, output_filename):
                 f.close()
 
             defFileContents = defTemplate % os.path.basename(output_filename)
-            defFilePath = os.path.join(output_dir, 'python_dll_def.def')
+            defFilePath = os.path.join(output_dir, 'infra', 'python_dll_def.def')
             f = file(defFilePath, 'wb')
             try:
                 f.write(defFileContents)

examples/hello/hello.d

@@ -8,8 +8,7 @@ void hello() {
     writefln("Hello, world!");
 }
 
-extern(C)
-export void inithello() {
+extern(C) void PydMain() {
     def!(hello);
-    module_init("hello");
+    module_init();
 }

examples/inherit/inherit.d

@@ -30,11 +30,10 @@ void call_poly(Base b) {
     b.foo();
 }
 
-extern(C)
-export void initinherit() {
+extern(C) void PydMain() {
     def!(call_poly);
 
-    module_init("inherit");
+    module_init();
 
     wrapped_class!(Base) b;
     b.def!(Base.foo);

examples/testdll/testdll.d

@@ -127,8 +127,9 @@ void throws() {
     throw new Exception("Yay! An exception!");
 }
 
-extern (C)
-export void inittestdll() {
+//extern (C)
+//export void inittestdll() {
+extern(C) void PydMain() {
     def!(foo);
     // Python does not support function overloading. This allows us to wrap
     // an overloading function under a different name. Note that if the
@@ -144,7 +145,7 @@ export void inittestdll() {
     def!(dg_test);
     def!(throws);
 
-    module_init("testdll");
+    module_init();
 
     wrapped_class!(Foo) f;
     // Constructor wrapping

html_doc/basics.html

@@ -33,9 +33,8 @@ <h3>Module basics</h3>
 
 <pre class="code"><span class="keyword">import</span> pyd.pyd;
 
-<span class="keyword">extern</span> (C)
-<span class="keyword">export void</span> inittestmodule() {
-    module_init(<span class="string">"testmodule"</span>);
+<span class="keyword">extern</span> (C) <span class="keyword">void</span> PydMain() {
+    module_init();
 }</pre>
 
 <p>The first line imports Pyd:</p>
@@ -44,17 +43,15 @@ <h3>Module basics</h3>
 
 <p>The <code>pyd</code> module in the <code>pyd</code> package publicly imports all of the other components of Pyd.</p>
 
-<p>The "init" function is a requirement of the Python/C API. It is a global function with the footprint <code>extern(C) export void function()</code>. Its name <em>must</em> be <code>init</code> plus the name of your module. This function must then contain a call to <code>module_init</code>, with the same module name as an argument. (Users of Boost.Python will be familiar with the <code>BOOST_PYTHON_MODULE</code> macro used in that library. Unfortunately for this purpose, D has no preprocessor, and cannot define the name of a function like the C preprocessor can. Therefore, users of Pyd must define their init functions manually.)</p>
+<p>The <code>PydMain</code> function is called when the module is imported by Python. You will call most of Pyd's API from here. At the very least, <code>PydMain</code> <em>must</em> contain a call to <code>module_init</code>. The <code>module_init</code> function has the following form:</p>
 
-<p>The <code>module_init</code> function has the following form:</p>
-
-<p><code>PyObject* module_init(char[] <span class="arg">name</span>, char[] <span class="arg">docstring</span>="");</code></p>
+<p><code>PyObject* module_init(char[] <span class="arg">docstring</span>="");</code></p>
 
 <p>It does little more than call <a href="http://docs.python.org/api/allocating-objects.html">Py_InitModule</a> and return the new module object. This object is also available via the <code>Pyd_Module_p</code> property once you've called <code>module_init</code>.</p>
 
-<p>Due to the way in which Pyd implements function and class wrapping, any calls to <code>def</code> must occur <em>before</em> the call to <code>module_init</code>, and any calls to <code>finalize_class</code> must occur <em>after</em> the call. I know this seems like a rather arbitrary rule, but it is important. Calls to <code>def</code> in the wrong place will simply be ignored, and calls to <code>finalize_class</code> in the wrong place will throw an assert. (And this assert will cause the Python interpreter to crash. So be warned.)</p>
+<p>Due to the way in which Pyd implements function and class wrapping, any calls to <code>def</code> must occur <em>before</em> the call to <code>module_init</code>, and any calls to <code>finalize_class</code> must occur <em>after</em> the call. I know this seems like a rather arbitrary rule, but it is important. Calls to <code>def</code> in the wrong place will simply be ignored, and calls to <code>finalize_class</code> in the wrong place will throw an assert.</p>
 
-<p>It is important that no D exceptions escape from the init function. If you put any of your own code in the init function, be sure to guard it aginst exceptions. The <a href="except_wrapping.html">exception wrapping</a> facilities of Pyd provide some useful functions for this purpose.</p>
+<p><code>PydMain</code> will catch any D exception that is thrown from inside it, and <a href="except_wrapping.html">safely pass that exception to Python</a>.</p>
 </div>
 
 </body>

html_doc/celerid.html

@@ -54,10 +54,11 @@ <h1>CeleriD</h1>
     <dl>
     <dt><code>version_flags</code></dt> <dd>This should be a list of strings, which will be passed to the D compiler as version flags.</dd>
     <dt><code>debug_flags</code></dt> <dd>Similar to <code>version_flags</code>, the strings in this list will be passed to D as debug flags.</dd>
-    <dt><code>raw_only</code></dt> <dd>This flag defaults to <code>False</code>. When <code>True</code>, it supresses the compilation and linkage of Pyd, StackThreads, and meta. This is useful if you only want to write a raw Python/C extension without the overhead of Pyd and its auxiliary packages. This is equivalent to specifying <code>False</code> to the next three flags.</dd>
+    <dt><code>raw_only</code></dt> <dd>This flag defaults to <code>False</code>. When <code>True</code>, it supresses the compilation and linkage of Pyd, StackThreads, and meta. This is useful if you only want to write a raw Python/C extension without the overhead of Pyd and its auxiliary packages. This is equivalent to specifying <code>False</code> to the next four flags.</dd>
     <dt><code>with_pyd</code></dt> <dd>This flag defaults to <code>True</code>. When <code>False</code>, it supresses the compilation and linkage of Pyd. This is useful if you want to write a raw Python/C extension and don't want the overhead of compiling Pyd.</dd>
     <dt><code>with_st</code></dt> <dd>This flag defaults to <code>True</code>. When <code>False</code>, it supresses the compilation and linkage of StackThreads. Pyd uses StackThreads for its iteration wrapping support. By setting this to <code>False</code>, opApply wrapping, <code>wrapped_class.iter</code>, and <code>wrapped_class.alt_iter</code> will be unavailable. If <code>with_pyd</code> and this are <code>True</code>, then the <code>Pyd_with_StackThreads</code> version flag will be defined (which is used internally by Pyd). <b>Important note:</b> StackThreads does not currently work with GDC! CeleriD will always set this flag to <code>False</code> when using GDC! This means that opApply wrapping is not available on Linux at this time.</dd>
     <dt><code>with_meta</code></dt> <dd>This flag defaults to <code>True</code>. When <code>False</code>, it supresses the compilation and linkage of <code>meta</code> (Pyd's metaprogramming package). Because Pyd depends on meta, an exception will be raised if <code>with_pyd</code> is <code>True</code> and this is not.</dd>
+    <dt><code>with_main</code></dt> <dd>This flag defaults to <code>True</code>. When <code>False</code>, it supresses the use of the "magic" <code>PydMain</code> function. (Instead, users must manually declare a C-style <code>init</code> function.) Do not use this unless you know what you are doing. If <code>with_pyd</code> is <code>False</code>, this will silently be set to <code>False</code> as well. <code>PydMain</code> can only be used if Pyd itself is in use.</dd>
     </dl>
 </dd>
 </dl>

html_doc/class_wrapping.html

@@ -120,7 +120,7 @@ <h3><a class="anchor" name="inheritance">Inheritance</a></h3>
     <span class="keyword">void</span> foo() { writefln(<span class="string">"Derived.foo"</span>); }
 }</pre>
 
-<p>These would be exposed to Python by putting this code in out init function after the call to <code>module_init</code>:</p>
+<p>These would be exposed to Python by putting this code in <code>PydMain</code> after the call to <code>module_init</code>:</p>
 
 <pre class="code">wrapped_class!(Base) b;
 b.def!(Base.foo);
@@ -225,7 +225,7 @@ <h3><a class="anchor" name="examples">Examples</a></h3>
     }
 }</pre>
 
-<p>We would expose this class to Python by putting this code in our init function after the call to <code>module_init</code>:</p>
+<p>We would expose this class to Python by putting this code in <code>PydMain</code> after the call to <code>module_init</code>:</p>
 
 <pre class="code"><span class="comment">// Make an instance of wrapped_class</span>
 wrapped_class!(Foo) f;

html_doc/func_wrapping.html

@@ -59,8 +59,7 @@ <h1>Function wrapping</h1>
     writefln(<span class="string">"i = %s\ns = %s"</span>, i, s);
 }
 
-<span class="keyword">extern</span> (C)
-<span class="keyword">export void</span> inittestmodule() {
+<span class="keyword">extern</span> (C) <span class="keyword">void</span> PydMain() {
     <span class="comment">// Plain old function</span>
     def!(foo);
     <span class="comment">// Wraps the lexically first function under the given name</span>
@@ -70,7 +69,7 @@ <h1>Function wrapping</h1>
     <span class="comment">// Wraps the function with default arguments</span>
     def!(baz);
 
-    module_init(<span class="string">"testmodule"</span>);
+    module_init();
 }</pre>
 
 <p>And when used in Python:</p>

html_doc/index.html

@@ -32,10 +32,6 @@ <h1>Pyd</h1>
 
 <p>Pyd includes the <code>meta.Nameof</code> package by Don Clugston. The source files meta.Nameof and meta.Demangle are copyright &copy; 2005-2006 Don Clugston.</p>
 
-<!-- Not documented, yet.
-<p>Pyd includes its own metaprogramming library, <code>meta</code>, which was written by Tomasz Stachowiak, Don Clugston, and Kirk McDonald. The <code>meta</code> library is documented <a href="meta/index.html">here</a>.</code>
--->
-
 <p>Pyd's Trac page can be found <a href="http://dsource.org/projects/pyd/wiki">here</a>.</p>
 
 <p>A simple "hello, world" module might look like this:</p>
@@ -47,11 +43,9 @@ <h1>Pyd</h1>
     writefln(<span class="string">"Hello, world!"</span>);
 }
 
-<span class="keyword">extern</span> (C)
-<span class="keyword">export void</span> inittestdll() {
+<span class="keyword">extern</span> (C) <span class="keyword">void</span> PydMain() {
     def!(hello_func);
-
-    module_init(<span class="string">"testdll"</span>);
+    module_init();
 }</pre>
 
 <p>When <a href="celerid.html">compiled</a>, the module can be loaded and used from Python like any other module:</p>

infrastructure/d/pydmain_template.d

@@ -0,0 +1,13 @@
+import pyd.def;
+import pyd.exception;
+
+extern(C) void PydMain();
+
+extern(C)
+export void init%(modulename)s() {
+    pyd.exception.exception_catcher(delegate void() {
+        pyd.def.pyd_module_name = "%(modulename)s";
+        PydMain();
+    });
+}
+

infrastructure/pyd/def.d

@@ -98,11 +98,14 @@ void def(char[] modulename, alias fn, char[] name = symbolnameof!(fn), fn_t=type
     (*list) ~= empty;
 }
 
+char[] pyd_module_name;
+
 /**
  * Module initialization function. Should be called after the last call to def.
  */
-PyObject* module_init(char[] name, char[] docstring="") {
+PyObject* module_init(char[] docstring="") {
     //_loadPythonSupport();
+    char[] name = pyd_module_name;
     ready_module_methods("");
     pyd_modules[""] = Py_InitModule3((name ~ \0).ptr, module_methods[""].ptr, (docstring ~ \0).ptr);
     return pyd_modules[""];