diff --git a/.circleci/config.yml b/.circleci/config.yml
index b8d5176cb..13673b5e0 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -14,6 +14,17 @@ workflows:
           node_version: latest
       - release-management/test-package:
           name: node-12
+      - release-management/test-nut:
+          name: nuts-on-linux
+          sfdx_version: latest
+          requires:
+            - node-latest
+      - release-management/test-nut:
+          name: nuts-on-windows
+          sfdx_version: latest
+          os: windows
+          requires:
+            - node-latest
       - release-management/release-package:
           sign: true
           github-release: true
diff --git a/command-snapshot.json b/command-snapshot.json
index 42e1182a0..dd4119839 100644
--- a/command-snapshot.json
+++ b/command-snapshot.json
@@ -2,18 +2,7 @@
   {
     "command": "force:source:convert",
     "plugin": "@salesforce/plugin-source",
-    "flags": [
-      "apiversion",
-      "json",
-      "loglevel",
-      "manifest",
-      "metadata",
-      "outputdir",
-      "packagename",
-      "rootdir",
-      "sourcepath",
-      "targetusername"
-    ]
+    "flags": ["json", "loglevel", "manifest", "metadata", "outputdir", "packagename", "rootdir", "sourcepath"]
   },
   {
     "command": "force:source:deploy",
diff --git a/messages/convert.json b/messages/convert.json
index 15ab594df..72c8d2a3a 100644
--- a/messages/convert.json
+++ b/messages/convert.json
@@ -12,5 +12,6 @@
     "sourcepath": "comma-separated list of paths to the local source files to convert",
     "metadata": "comma-separated list of metadata component names to convert"
   },
-  "success": "Source was successfully converted to Metadata API format and written to the location: %s"
+  "success": "Source was successfully converted to Metadata API format and written to the location: %s",
+  "convertFailed": "Failed to convert source"
 }
diff --git a/package.json b/package.json
index 8ccfe178e..053977022 100644
--- a/package.json
+++ b/package.json
@@ -120,7 +120,8 @@
     "test": "sf-test",
     "test:command-reference": "./bin/run commandreference:generate --erroronwarnings",
     "test:deprecation-policy": "./bin/run snapshot:compare",
-    "test:nuts": "ts-node ./test/nuts/generateNuts.ts && nyc mocha \"**/*.nut.ts\" --slow 3000 --timeout 600000 --parallel --retries 0",
+    "test:nuts": "ts-node ./test/nuts/generateNuts.ts && nyc mocha \"**/convert.*.nut.ts\" --slow 3000 --timeout 600000 --parallel --retries 0",
+    "test:nuts:convert": "ts-node ./test/nuts/generateNuts.ts && nyc mocha \"**/convert.*.nut.ts\" --slow 3000 --timeout 600000 --parallel --retries 0",
     "version": "oclif-dev readme"
   },
   "husky": {
diff --git a/src/commands/force/source/convert.ts b/src/commands/force/source/convert.ts
index 5646b7fb7..b35ba7791 100644
--- a/src/commands/force/source/convert.ts
+++ b/src/commands/force/source/convert.ts
@@ -21,7 +21,6 @@ export class Convert extends SourceCommand {
   public static readonly description = messages.getMessage('description');
   public static readonly examples = messages.getMessage('examples').split(os.EOL);
   public static readonly requiresProject = true;
-  public static readonly requiresUsername = true;
   public static readonly flagsConfig: FlagsConfig = {
     rootdir: flags.directory({
       char: 'r',
diff --git a/src/formatters/convertResultFormatter.ts b/src/formatters/convertResultFormatter.ts
index 5132ea5a7..c6fe17710 100644
--- a/src/formatters/convertResultFormatter.ts
+++ b/src/formatters/convertResultFormatter.ts
@@ -7,7 +7,7 @@
 
 import { resolve } from 'path';
 import { UX } from '@salesforce/command';
-import { Logger, Messages } from '@salesforce/core';
+import { Logger, Messages, SfdxError } from '@salesforce/core';
 import { ConvertResult } from '@salesforce/source-deploy-retrieve';
 import { ResultFormatter } from './resultFormatter';
 
@@ -36,8 +36,7 @@ export class ConvertResultFormatter extends ResultFormatter {
     if (this.isSuccess()) {
       this.ux.log(messages.getMessage('success', [this.result.packagePath]));
     } else {
-      // TODO: make this better
-      this.ux.log('Failed to convert source');
+      throw new SfdxError(messages.getMessage('convertFailed'), 'ConvertFailed');
     }
   }
 }
diff --git a/test/nuts/assertions.ts b/test/nuts/assertions.ts
index fe767cf2e..0faf96b9e 100644
--- a/test/nuts/assertions.ts
+++ b/test/nuts/assertions.ts
@@ -137,6 +137,7 @@ export class Assertions {
    * Expects files to exist in convert output directory
    */
   public async filesToBeConverted(directory: string, globs: string[]): Promise<void> {
+    directory = directory.split(path.sep).join('/');
     const fullGlobs = globs.map((glob) => [directory, glob].join('/'));
     const convertedFiles = await fg(fullGlobs);
     expect(convertedFiles.length, 'files to be converted').to.be.greaterThan(0);
diff --git a/test/nuts/generateNuts.ts b/test/nuts/generateNuts.ts
index 2231fe329..4feec23ef 100644
--- a/test/nuts/generateNuts.ts
+++ b/test/nuts/generateNuts.ts
@@ -6,6 +6,7 @@
  */
 
 import * as path from 'path';
+import * as os from 'os';
 import { fs } from '@salesforce/core';
 import { EXECUTABLES, TEST_REPOS_MAP, RepoConfig } from './testMatrix';
 
@@ -38,6 +39,12 @@ async function generateNut(
     ? `${seedName}.${repoName}.${executableName}.nut.ts`
     : `${seedName}.${executableName}.nut.ts`;
   const nutFilePath = path.join(generatedDir, nutFileName);
+
+  // On windows the executable path is being changed to
+  // single backslashes so ensure proper path.sep.
+  if (os.platform() === 'win32') {
+    executable = executable.replace(/\\/g, '\\\\');
+  }
   const contents = seedContents
     .replace(/%REPO_URL%/g, repo?.gitUrl)
     .replace(/%EXECUTABLE%/g, executable)
@@ -47,14 +54,14 @@ async function generateNut(
 
 async function generateNuts(): Promise<void> {
   const generatedDir = path.resolve(__dirname, 'generated');
-  await fs.rmdir(generatedDir, { recursive: true });
+  fs.rmSync(generatedDir, { force: true, recursive: true });
   await fs.mkdirp(generatedDir);
   const seeds = await getSeedFiles();
   for (const seed of seeds) {
     const seedName = path.basename(seed).replace('.seed.ts', '');
     const seedContents = await fs.readFile(seed, 'UTF-8');
     for (const executable of EXECUTABLES.filter((e) => !e.skip)) {
-      const hasRepo = /const\sREPO\s=\s(.*?)\n/.test(seedContents);
+      const hasRepo = /const\sREPO\s=\s/.test(seedContents);
       if (hasRepo) {
         for (const repo of [...TEST_REPOS_MAP.values()].filter((r) => !r.skip)) {
           await generateNut(generatedDir, seedName, seedContents, executable.path, repo);
diff --git a/test/nuts/nutshell.ts b/test/nuts/nutshell.ts
index 47e657b35..a264242df 100644
--- a/test/nuts/nutshell.ts
+++ b/test/nuts/nutshell.ts
@@ -371,7 +371,7 @@ export class Nutshell extends AsyncCreatable<Nutshell.Options> {
   }
 
   protected async init(): Promise<void> {
-    if (!Nutshell.Env.getString('TESTKIT_HUB_USERNAME')) {
+    if (!Nutshell.Env.getString('TESTKIT_HUB_USERNAME') && !Nutshell.Env.getString('TESTKIT_AUTH_URL')) {
       ensureString(Nutshell.Env.getString('TESTKIT_JWT_KEY'));
       ensureString(Nutshell.Env.getString('TESTKIT_JWT_CLIENT_ID'));
       ensureString(Nutshell.Env.getString('TESTKIT_HUB_INSTANCE'));
diff --git a/test/nuts/seeds/convert.seed.ts b/test/nuts/seeds/convert.seed.ts
index ae8ae71df..30a499885 100644
--- a/test/nuts/seeds/convert.seed.ts
+++ b/test/nuts/seeds/convert.seed.ts
@@ -6,6 +6,8 @@
  */
 
 import * as path from 'path';
+import * as shelljs from 'shelljs';
+import { asString } from '@salesforce/ts-types';
 import { Nutshell } from '../nutshell';
 import { TEST_REPOS_MAP } from '../testMatrix';
 
@@ -13,6 +15,22 @@ import { TEST_REPOS_MAP } from '../testMatrix';
 const REPO = TEST_REPOS_MAP.get('%REPO_URL%');
 const EXECUTABLE = '%EXECUTABLE%';
 
+// SDR does not output the package.xml in the same location as toolbelt
+// so we have to find it within the output dir, move it, and delete the
+// generated dir.
+const mvManifest = (dir: string) => {
+  const manifest = shelljs.find(dir).filter((file) => file.endsWith('package.xml'));
+  if (!manifest?.length) {
+    throw Error(`Did not find package.xml within ${dir}`);
+  }
+  shelljs.mv(manifest[0], path.join(process.cwd()));
+  shelljs.rm('-rf', dir);
+};
+
+const isSourcePlugin = (): boolean => {
+  return EXECUTABLE.endsWith(`${path.sep}bin${path.sep}run`);
+};
+
 context('Convert NUTs [name: %REPO_NAME%] [exec: %EXECUTABLE%]', () => {
   let nutshell: Nutshell;
 
@@ -30,15 +48,32 @@ context('Convert NUTs [name: %REPO_NAME%] [exec: %EXECUTABLE%]', () => {
   });
 
   describe('--manifest flag', () => {
+    let convertDir: string;
+
     for (const testCase of REPO.convert.manifest) {
       it(`should convert ${testCase.toConvert}`, async () => {
-        await nutshell.convert({ args: `--sourcepath ${testCase.toConvert} --outputdir out1` });
-        const packageXml = path.join('out1', 'package.xml');
+        // Generate a package.xml by converting via sourcepath
+        const toConvert = path.normalize(testCase.toConvert);
+        await nutshell.convert({
+          args: `--sourcepath ${toConvert} --outputdir out1`,
+          exitCode: 0,
+        });
+        const outputDir = path.join(process.cwd(), 'out1');
+        mvManifest(outputDir);
+        const packageXml = path.join(process.cwd(), 'package.xml');
+
+        const res = await nutshell.convert({ args: `--manifest ${packageXml} --outputdir out2`, exitCode: 0 });
 
-        await nutshell.convert({ args: `--manifest ${packageXml} --outputdir out2` });
-        await nutshell.expect.directoryToHaveSomeFiles('out2');
-        await nutshell.expect.fileToExist(path.join('out2', 'package.xml'));
-        await nutshell.expect.filesToBeConverted('out2', testCase.toVerify);
+        convertDir = path.relative(process.cwd(), asString(res.result?.location));
+        await nutshell.expect.directoryToHaveSomeFiles(convertDir);
+        await nutshell.expect.fileToExist(path.join(convertDir, 'package.xml'));
+        await nutshell.expect.filesToBeConverted(convertDir, testCase.toVerify);
+      });
+
+      afterEach(() => {
+        if (convertDir) {
+          shelljs.rm('-rf', convertDir);
+        }
       });
     }
 
@@ -49,34 +84,57 @@ context('Convert NUTs [name: %REPO_NAME%] [exec: %EXECUTABLE%]', () => {
   });
 
   describe('--metadata flag', () => {
+    let convertDir: string;
+
     for (const testCase of REPO.convert.metadata) {
       it(`should convert ${testCase.toConvert}`, async () => {
-        await nutshell.convert({ args: `--metadata ${testCase.toConvert} --outputdir out` });
-        await nutshell.expect.directoryToHaveSomeFiles('out');
-        await nutshell.expect.fileToExist(path.join('out', 'package.xml'));
-        await nutshell.expect.filesToBeConverted('out', testCase.toVerify);
+        const res = await nutshell.convert({ args: `--metadata ${testCase.toConvert} --outputdir out`, exitCode: 0 });
+
+        convertDir = path.relative(process.cwd(), asString(res.result?.location));
+        await nutshell.expect.directoryToHaveSomeFiles(convertDir);
+        await nutshell.expect.fileToExist(path.join(convertDir, 'package.xml'));
+        await nutshell.expect.filesToBeConverted(convertDir, testCase.toVerify);
       });
     }
 
+    afterEach(() => {
+      if (convertDir) {
+        shelljs.rm('-rf', convertDir);
+      }
+    });
+
     it('should throw an error if the metadata is not valid', async () => {
       const convert = await nutshell.convert({ args: '--metadata DOES_NOT_EXIST', exitCode: 1 });
-      nutshell.expect.errorToHaveName(convert, 'UnsupportedType');
+      const expectedError = isSourcePlugin() ? 'RegistryError' : 'UnsupportedType';
+      nutshell.expect.errorToHaveName(convert, expectedError);
     });
   });
 
   describe('--sourcepath flag', () => {
+    let convertDir: string;
+
     for (const testCase of REPO.convert.sourcepath) {
       it(`should convert ${testCase.toConvert}`, async () => {
-        await nutshell.convert({ args: `--sourcepath ${testCase.toConvert} --outputdir out` });
-        await nutshell.expect.directoryToHaveSomeFiles('out');
-        await nutshell.expect.fileToExist(path.join('out', 'package.xml'));
-        await nutshell.expect.filesToBeConverted('out', testCase.toVerify);
+        const toConvert = path.normalize(testCase.toConvert);
+        const res = await nutshell.convert({ args: `--sourcepath ${toConvert} --outputdir out`, exitCode: 0 });
+
+        convertDir = path.relative(process.cwd(), asString(res.result?.location));
+        await nutshell.expect.directoryToHaveSomeFiles(convertDir);
+        await nutshell.expect.fileToExist(path.join(convertDir, 'package.xml'));
+        await nutshell.expect.filesToBeConverted(convertDir, testCase.toVerify);
       });
     }
 
+    afterEach(() => {
+      if (convertDir) {
+        shelljs.rm('-rf', convertDir);
+      }
+    });
+
     it('should throw an error if the sourcepath is not valid', async () => {
       const convert = await nutshell.convert({ args: '--sourcepath DOES_NOT_EXIST', exitCode: 1 });
-      nutshell.expect.errorToHaveName(convert, 'SourcePathInvalid');
+      const expectedError = isSourcePlugin() ? 'SfdxError' : 'SourcePathInvalid';
+      nutshell.expect.errorToHaveName(convert, expectedError);
     });
   });
 });
diff --git a/test/nuts/testMatrix.ts b/test/nuts/testMatrix.ts
index b869cb69f..8fb832d8d 100644
--- a/test/nuts/testMatrix.ts
+++ b/test/nuts/testMatrix.ts
@@ -22,7 +22,7 @@ export const EXECUTABLES = [
   },
   {
     path: path.join(process.cwd(), 'bin', 'run'), // path to the plugin's bin/run executable
-    skip: !env.getBoolean('PLUGIN_SOURCE_TEST_BIN_RUN', false),
+    skip: !env.getBoolean('PLUGIN_SOURCE_TEST_BIN_RUN', true),
   },
 ];