Get CI working

Update `@types/ms` resolution to version 0.7.32 in `package.json` and `yarn.lock`.

Add ts-node script and update linting commands in package.json and lint.sh.

Enhance CI workflow with debug info for PR events; add new blog post template and assets.

Refactor types in find-markdown.ts and add test PR event JSON file.

Refactor PR file handling: update to use 'files' instead of 'added_files' for added file tracking in GitHub Actions. Adjust test data accordingly.

Remove debug logging and add base branch environment variable in workflows

Enhance GitHub Actions workflow: add branch info echo and ensure make step before linting.

Refactor linting scripts and update pull request workflow

- Removed lint step from GitHub Actions workflow
- Simplified the `find-markdown.ts` script by removing GHA-related functionality
- Updated `getGitModifiedFiles` to accept a base branch parameter
- Resolved types and streamlined logic for markdown file detection
- Adjusted lint scripts to handle modes more effectively

Update markdown linting script to exclude 'node_modules' directory and add debug logging functionality in the linting process. Set DEBUG mode to off by default.

Update GitHub Actions workflow and improve file detection logic

- Upgraded `actions/checkout` from v2 to v4 and set fetch depth.
- Enhanced `getGitModifiedFiles` function to handle both local and GitHub Actions environments.
- Improved error handling and added debug logging for file detection.

Refactor GitHub Actions workflow and TypeScript script for enhanced efficiency and clarity in fetching branches and modified files.

Author: adamgordonbell

.github/workflows/pull-request.yml

@@ -19,12 +19,35 @@ jobs:
     environment: testing
     steps:
 
-      - uses: actions/checkout@v2
-
+      - uses: actions/checkout@v4
+        with:
+          # Only fetch PR branch
+          fetch-depth: 1
+          ref: ${{ github.event.pull_request.head.sha }}
+      
+      - name: Fetch base branch
+        run: |
+          # Fetch and create local master branch
+          git -c protocol.version=2 fetch --no-tags --prune --progress --no-recurse-submodules --depth=1 origin master:master
+          # Verify branches
+          echo "Local branches:"
+          git branch -v
+          echo "\nRemote branches:"
+          git branch -r
+      
+      - name: Set base branch env
+        run: |
+          if [ "${{ github.event_name }}" = "pull_request" ]; then
+            echo "BASE_BRANCH=${{ github.event.pull_request.base.ref }}" >> $GITHUB_ENV
+          else
+            # This covers push scenario to master
+            # or any other scenario with no PR info
+            echo "BASE_BRANCH=" >> $GITHUB_ENV
+          fi
+          echo "Branch:${{ github.event.pull_request.base.ref }}"
       - uses: actions/setup-node@v1
         with:
           node-version: '18.x'
-
       - uses: peaceiris/actions-hugo@v2
         with:
           hugo-version: '0.135.0'
@@ -55,6 +78,7 @@ jobs:
           ALGOLIA_APP_SEARCH_KEY: ${{ vars.ALGOLIA_APP_SEARCH_KEY }}
           ALGOLIA_APP_ADMIN_KEY: ${{  secrets.ALGOLIA_APP_ADMIN_KEY }}
           NODE_OPTIONS: "--max_old_space_size=8192"
+          BASE_BRANCH: ${{ env.BASE_BRANCH }}
 
       - name: Archive test results
         uses: actions/upload-artifact@v4

content/blog/sfsdfsfs/index.md

@@ -0,0 +1,85 @@
+---
+title: "Sfsdfsfs"
+
+# The date represents the post's publish date, and by default corresponds with
+# the date and time this file was generated. Dates are used for display and
+# ordering purposes only; they have no effect on whether or when a post is
+# published. To influence the ordering of posts published on the same date, use
+# the time portion of the date value; posts are sorted in descending order by
+# date/time.
+date: 2025-02-04T10:14:44-05:00
+
+# The draft setting determines whether a post is published. Set it to true if
+# you want to be able to merge the post without publishing it.
+draft: false
+
+# Use the meta_desc property to provide a brief summary (one or two sentences)
+# of the content of the post, which is useful for targeting search results or
+# social-media previews. This field is required or the build will fail the
+# linter test. Max length is 160 characters.
+meta_desc:
+
+# The meta_image appears in social-media previews and on the blog home page. A
+# placeholder image representing the recommended format, dimensions and aspect
+# ratio has been provided for you.
+meta_image: meta.png
+
+# At least one author is required. The values in this list correspond with the
+# `id` properties of the team member files at /data/team/team. Create a file for
+# yourself if you don't already have one.
+authors:
+    - joe-duffy
+
+# At least one tag is required. Lowercase, hyphen-delimited is recommended.
+tags:
+    - change-me
+
+
+# The social copy used to promote this post on Twitter and Linkedin. These
+# properties do not actually create the post and have no effect on the
+# generated blog page. They are here strictly for reference.
+
+# Here are some examples of posts we have made in the past for inspiration:
+# https://www.linkedin.com/feed/update/urn:li:activity:7171191945841561601
+# https://www.linkedin.com/feed/update/urn:li:activity:7169021002394296320
+# https://www.linkedin.com/feed/update/urn:li:activity:7155606616455737345
+# https://twitter.com/PulumiCorp/status/1763265391042654623
+# https://twitter.com/PulumiCorp/status/1762900472489185492
+# https://twitter.com/PulumiCorp/status/1755637618631405655
+
+social:
+    twitter:
+    linkedin:
+
+# See the blogging docs at https://github.com/pulumi/docs/blob/master/BLOGGING.md
+# for details, and please remove these comments before submitting for review.
+---
+
+What you put here will appear on the index page. In most cases, you'll also want to add a Read More link after this paragraph (though technically, that's optional. To do that, just add an HTML comment like the one below.
+
+<!--more-->
+
+And then everything _after_ that comment will appear on the post page itself.
+
+Either way, avoid using images or code samples [in the first 70 words](https://gohugo.io/content-management/summaries/#automatic-summary-splitting) of your post, as these may not render properly in summary contexts (e.g., on the blog home page or in social-media previews).
+
+## Writing the Post
+
+For help assembling the content of your post, see [BLOGGING.md](https://github.com/pulumi/docs/blob/master/BLOGGING.md). For general formatting guidelines, see the [Style Guide](https://github.com/pulumi/docs/blob/master/STYLE-GUIDE.md).
+
+## Code Samples
+
+```typescript
+let bucket = new aws.s3.Bucket("stuff");
+...
+```
+
+## Images
+
+![Placeholder Image](meta.png)
+
+## Videos
+
+{{< youtube "kDB-YRKFfYE?rel=0" >}}
+
+Note the `?rel=0` param, which tells YouTube to suggest only videos from same channel.

content/blog/sfsdfsfs/meta.png

@@ -5,7 +5,8 @@
   "license": "Apache-2.0",
   "scripts": {
     "minify-css": "node scripts/minify-css.js",
-    "prepare": "husky"
+    "prepare": "husky",
+    "ts-node": "ts-node"
   },
   "dependencies": {
     "@aws-sdk/client-s3": "^3.427.0",
@@ -40,6 +41,9 @@
     "zod": "^3.24.1",
     "zod-matter": "^0.1.1"
   },
+  "resolutions": {
+    "@types/ms": "0.7.32"
+  },
   "devDependencies": {
     "@types/js-yaml": "^4.0.9",
     "@types/markdownlint": "^0.18.0",

package.json

@@ -29,7 +29,7 @@ export interface MarkdownFile {
 /**
  * Different modes for finding markdown files and determining if they're new
  */
-type FinderMode = 'git' | 'gha' | 'standard';
+export type FinderMode = 'git' | 'gha';
 
 const AUTO_GENERATED_HEADING_REGEX = /^> This page was automatically generated\./m;
 
@@ -69,54 +69,97 @@ function shouldExcludeByFrontMatter(frontMatter: FrontMatter | undefined): boole
 import { debug } from './utils';
 
 /**
- * Get list of files modified in the latest git commit
+ * Get list of files that are newly added in the current branch compared to base branch,
+ * including staged and unstaged changes. Handles both local git and GitHub Actions environments.
  */
-function getGitModifiedFiles(): Set<string> {
+function getGitModifiedFiles(baseBranch?: string): Set<string> {
     try {
         debug('Getting git modified files...');
-        const output = execSync('git diff-tree --no-commit-id --name-only -r HEAD', { encoding: 'utf8' });
-        const set = new Set(output.split('\n').filter(Boolean));
-        debug('Modified files:', set);
-        return set;
-    } catch (error) {
-        console.warn('Failed to get git modified files, treating all files as existing');
-        return new Set<string>();
-    }
-}
+        const newFiles = new Set<string>();
+
+        // In GitHub Actions, use FETCH_HEAD since we just fetched the base branch
+        const isGHA = process.env.GITHUB_ACTIONS === 'true';
+        const baseRef = isGHA ? 'FETCH_HEAD' : (baseBranch || process.env.BASE_BRANCH || 'master');
+        debug('Using base ref:', baseRef);
+
+        try {
+            // In GHA, we've just fetched the base branch to FETCH_HEAD, so this is safe
+            const diffOutput = execSync(`git diff --name-status HEAD ${baseRef}`, { encoding: 'utf8' });
+            debug('Got diff output');
+            diffOutput.split('\n')
+                .filter(line => line.startsWith('A\t'))
+                .map(line => line.split('\t')[1])
+                .forEach(file => newFiles.add(file));
+        } catch (error) {
+            debug('Diff failed:', error);
+            if (isGHA) {
+                // In GHA, if diff fails, something is wrong with our setup
+                throw error;
+            }
+            // In local dev, try to get staged/untracked files
+            debug('Falling back to staged/untracked files only');
+        }
 
-/**
- * Get list of files modified in the current PR (GitHub Actions)
- */
-function getGHAModifiedFiles(): Set<string> {
-    const eventPath = process.env.GITHUB_EVENT_PATH;
-    if (!eventPath) {
-        console.warn('No GITHUB_EVENT_PATH found, treating all files as existing');
-        return new Set<string>();
-    }
+        // In local dev, also look for staged and untracked files
+        if (!isGHA) {
+            try {
+                // Get new files from staged changes
+                const stagedFiles = execSync('git diff --name-status --cached', { encoding: 'utf8' });
+                stagedFiles.split('\n')
+                    .filter(line => line.startsWith('A\t'))
+                    .map(line => line.split('\t')[1])
+                    .forEach(file => newFiles.add(file));
+
+                // Get untracked files
+                debug('Getting untracked files...');
+                const untrackedFiles = execSync('git ls-files --others --exclude-standard --full-name', { encoding: 'utf8' });
+                debug('Raw untracked files output:', untrackedFiles);
+                untrackedFiles.split('\n').filter(Boolean).forEach(file => newFiles.add(file));
+
+                // Also get untracked markdown files specifically
+                const untrackedMarkdown = execSync('git ls-files --others --exclude-standard --full-name "*.md"', { encoding: 'utf8' });
+                debug('Raw untracked markdown files:', untrackedMarkdown);
+                untrackedMarkdown.split('\n').filter(Boolean).forEach(file => newFiles.add(file));
+            } catch (error) {
+                debug('Failed to get staged/untracked files:', error);
+            }
+        }
 
-    try {
-        const event = JSON.parse(fs.readFileSync(eventPath, 'utf8'));
-        const files = event.pull_request?.changed_files || [];
-        return new Set(files);
+        debug('Found new files:', newFiles);
+        return newFiles;
     } catch (error) {
-        console.warn('Failed to get PR modified files, treating all files as existing');
+        console.warn('Failed to get git modified files:', error);
+        if (process.env.GITHUB_ACTIONS === 'true') {
+            console.warn('Running in GitHub Actions - treating all files as existing');
+        } else {
+            console.warn('Running locally - treating all files as existing');
+        }
         return new Set<string>();
     }
 }
 
+
+
 /**
- * Find all markdown files in the given directory and its subdirectories.
+ * Find all markdown files in the given directory and its subdirectoriesA
  * Optionally marks files as new based on git/GHA context.
  */
 export function findMarkdownFiles(
     rootDir: string,
-    mode: FinderMode = 'standard',
-    excludePaths: string[] = ['/content/docs/reference/pkg', '/content/registry']
+    mode: FinderMode = 'git',
+    excludePaths: string[] = ['/content/docs/reference/pkg', '/content/registry', '/node_modules']
 ): MarkdownFile[] {
     const files: MarkdownFile[] = [];
-    const modifiedFiles = mode === 'git' ? getGitModifiedFiles() 
-                       : mode === 'gha' ? getGHAModifiedFiles()
-                       : new Set<string>();
+    
+    // Determine which files are considered new based on mode
+    let modifiedFiles = new Set<string>();
+    if (mode === 'git') {
+        modifiedFiles = getGitModifiedFiles();
+    } else if (mode === 'gha') {
+        const baseBranch = process.env.BASE_BRANCH;
+        debug('GHA mode using base branch:', baseBranch);
+        modifiedFiles = getGitModifiedFiles(baseBranch);
+    }
 
     function isExcluded(filePath: string): boolean {
         return excludePaths.some(excludePath => filePath.includes(excludePath));
@@ -142,9 +185,11 @@ export function findMarkdownFiles(
                 const content = fs.readFileSync(fullPath, 'utf8');
                 const frontMatter = parseFrontMatter(fullPath);
                 if (!shouldExcludeByFrontMatter(frontMatter)) {
+                    // Convert absolute path to relative for comparing with git modified files
+                    const relPath = path.relative(process.cwd(), fullPath);
                     files.push({
                         path: fullPath,
-                        isNew: modifiedFiles.has(fullPath),
+                        isNew: modifiedFiles.has(relPath),
                         content,
                         frontMatter
                     });
@@ -159,8 +204,19 @@ export function findMarkdownFiles(
 
 // If run directly, output found files
 if (require.main === module) {
-    const mode = (process.argv[2] as FinderMode) || 'standard';
-    const rootDir = process.argv[3] || '../../content';
-    const files = findMarkdownFiles(rootDir, mode);
-    console.log(JSON.stringify(files, null, 2));
+    const mode = (process.argv[2] as FinderMode) || 'gha';
+    const showAll = process.argv.includes('--show-all');
+    debug('Running in mode:', mode, 'show all:', showAll);
+    
+    // Find all markdown files
+    const files = findMarkdownFiles('.', mode);
+    
+    if (showAll) {
+        console.log('All markdown files (new files marked with *):')
+        console.log(JSON.stringify(files.map(f => `${f.isNew ? '*' : ' '} ${f.path}`), null, 2));
+    } else {
+        const newFiles = files.filter(f => f.isNew).map(f => f.path);
+        console.log('New markdown files detected:');
+        console.log(JSON.stringify(newFiles, null, 2));
+    }
 }

scripts/lint/find-markdown.ts

@@ -2,23 +2,35 @@
 
 set -o errexit -o pipefail
 
-# Get script directory
-SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
-
 DEBUG=0
 
+# Debug logging function
+debug() {
+    if [ -n "$DEBUG" ]; then
+        echo "[DEBUG] $1"
+    fi
+}
+
 # Determine mode based on environment
-if [ -n "$GITHUB_ACTIONS" ]; then
+if [ -n "$BASE_BRANCH" ]; then
     MODE="gha"
+    debug "Using base branch: $BASE_BRANCH"
 else
     MODE="git"
+    debug "Using git mode (no base branch)"
 fi
 
+# If in debug mode, show what files would be considered new
+# if [ -n "$DEBUG" ]; then
+#     debug "Preview of new files:"
+#     yarn ts-node "./scripts/lint/find-markdown.ts" "$MODE" --show-all
+# fi
+
 # Run markdown linting with debug flag
-DEBUG=$DEBUG ts-node "$SCRIPT_DIR/markdownlint-runner.ts" "$MODE"
+DEBUG=$DEBUG yarn ts-node "./scripts/lint/markdownlint-runner.ts" "$MODE"
 
 # Run front matter validation
-DEBUG=$DEBUG ts-node "$SCRIPT_DIR/validate-frontmatter.ts" "$MODE"
+DEBUG=$DEBUG yarn ts-node "./scripts/lint/validate-frontmatter.ts" "$MODE"
 
 # Run prettier
 yarn prettier --check .

scripts/lint/lint.sh

@@ -1,5 +1,5 @@
 import * as path from 'path';
-import { findMarkdownFiles } from './find-markdown';
+import { findMarkdownFiles, FinderMode } from './find-markdown';
 
 // Types for markdownlint
 type Token = {
@@ -61,7 +61,7 @@ function groupLintErrorOutput(result: Record<string, LintError[]>): LintResult[]
 import { debug } from './utils';
 
 // Get mode from command line args, defaulting to 'standard'
-const mode = (process.argv[2] as 'git' | 'gha' | 'standard') || 'standard';
+const mode = (process.argv[2] as FinderMode) || 'git';
 debug('Running in mode:', mode);
 
 // Get markdown files to lint