Anchors Drifting #157

latifs · 2022-10-05T23:13:36Z

Hi guys,
I have been trying to implement a fairly simple example using hit-test and anchors to display elements in the real world.
somehow I have been struggling with making anchors stick to the real world.
When I get close to the boxes(which are also anchors) they seem to be in place but as I step back they seems to drift pretty significantly.

I've attached a video to show the issue I'm facing and also a snippet of code. The question is simple am I missing something or is this expected beahvior?

Screen_Recording_20221005-155854_Chrome.mp4

<html>
  <head>
    <title>Hit Test Anchors</title>
    <meta name="description" content="Text Anchors - A-Frame" />
    <script src="https://aframe.io/releases/1.3.0/aframe.min.js"></script>
    <script>
      AFRAME.registerComponent('hit-test', {
        init: function () {
          this.viewerSpace = null;
          this.xrHitTestSource = null;
          this.hitTestResult = null;
          this.anchoredObjects = [];
          this.refSpace = null;
          this.isAnchorCaptured = false;
          this.mainAnchorPose = {};
          this.fixtures = [
            {
              origin: { fx: 0.485, fy: 0, fz: -2.7 },
              dimension: { height: 0.35, width: 0.35, depth: 0.3 },
              color: 'tomato',
              id: null,
            },
            {
              origin: { fx: -0.745, fy: 0, fz: -3.19 },
              dimension: { height: 0.4, width: 0.35, depth: 0.38 },
              color: 'blue',
              id: null,
            },

            // {
            //   origin: { fx: -0.61, fy: 0.135, fz: -5.34 },
            //   dimension: { height: 1.9, width: 0.82, depth: 0 },
            //   color: 'tomato',
            //   id: null,
            // },
            // {
            //   origin: { fx: 1.49, fy: 0, fz: -4.35 },
            //   dimension: { height: 2, width: 0.61, depth: 1.95 },
            //   color: 'green',
            //   id: null,
            // },
          ];

          this.anchorInfo = {
            origin: {},
            dimension: { height: 0.1, width: 0.1, depth: 0.1 },
            color: 'cyan',
            id: null,
          };

          // Listeners
          this.el.sceneEl.renderer.xr.addEventListener(
            'sessionstart',
            this.sessionStart.bind(this)
          );
        },

        sessionStart: async function () {
          this.session = this.el.sceneEl.renderer.xr.getSession();
          this.viewerSpace = await this.session.requestReferenceSpace('viewer');
          const hitTestSource = await this.session.requestHitTestSource({
            space: this.viewerSpace,
            entityTypes: ['plane'],
          });
          this.xrHitTestSource = hitTestSource;
          this.refSpace = await this.session.requestReferenceSpace(
            'local-floor'
          );
          // Listeners
          this.session.requestAnimationFrame(this.onXRFrame.bind(this));
          this.session.addEventListener('select', this.onSelect.bind(this));
        },

        createDOMEl: function (fixtureData) {
          let element = document.createElement('a-entity');
          const { dimension, color, id } = fixtureData;

          element.setAttribute(
            'geometry',
            `primitive:box;height:${dimension.height};width:${dimension.width};depth:${dimension.depth}`
          );
          element.setAttribute('material', `color:${color}`);
          element.setAttribute('id', id);
          return element;
        },

        createOffsetAnchor: async function (frame, fixtureData) {
          const { fx, fy, fz } = fixtureData.origin;
          const { x: ax, y: ay, z: az } = this.mainAnchorPose.position;
          const { x, y, z } = { x: ax + fx, y: ay, z: az + fz };

          let anchorFixture = await frame.createAnchor(
            new XRRigidTransform({ x, y, z }, { x: 0, y: 0, z: 0, w: 1 }),
            this.refSpace
          );

          this.anchoredObjects.push({
            id: fixtureData.id,
            anchor: anchorFixture,
          });
        },

        placeFixtures: function (frame) {
          for (let i = 0; i < this.fixtures.length; i++) {
            this.createOffsetAnchor(frame, this.fixtures[i]);

            const id = `fixture-${this.generateId()}`;
            this.fixtures[i].id = id;

            const domEl = this.createDOMEl(this.fixtures[i]);
            console.log('domEl:', domEl);

            this.el.sceneEl.append(domEl);
          }
        },

        generateId: function () {
          return `${parseInt(Math.random() * 100)}`;
        },

        onSelect: async function (evt) {
          let anchor = await this.hitTestResult.createAnchor();
          let id = `anchor-` + this.generateId();
          this.anchorInfo.id = id;
          let element = this.createDOMEl(this.anchorInfo);
          this.anchoredObjects.push({ id, anchor });
          this.el.sceneEl.append(element);
        },

        onXRFrame: function (t, frame) {
          const session = frame.session;
          const viewerPose = frame.getViewerPose(this.refSpace);

          // Update Reticle Position
          if (this.xrHitTestSource && viewerPose) {
            const hitTestResults = frame.getHitTestResults(
              this.xrHitTestSource
            );
            if (hitTestResults.length > 0) {
              let pose = hitTestResults[0].getPose(this.refSpace);
              this.el.setAttribute('position', pose.transform.position);
              this.hitTestResult = hitTestResults[0];
            }
          }

          // Update Anchors positions
          for (let index = 0; index < this.anchoredObjects.length; index++) {
            let { anchor, id } = this.anchoredObjects[index];

            if (frame.trackedAnchors.has(anchor)) {
              let anchorPose = frame.getPose(anchor.anchorSpace, this.refSpace);
              let element = document.querySelector(`#${id}`);

              if (element && anchorPose) {
                element.setAttribute('position', anchorPose.transform.position);

                if (!this.isAnchorCaptured) {
                  console.log('Anchor placed:', this.isAnchorCaptured);
                  this.isAnchorCaptured = true;
                  this.mainAnchorPose = anchorPose.transform;
                  this.placeFixtures(frame);
                }
              }
            }
          }

          this.session.requestAnimationFrame(this.onXRFrame.bind(this));
        },
      });
    </script>
  </head>
  <body>
    <a-scene
      device-orientation-permission-ui="enabled: true"
      material="opacity: 0.0; transparent: true"
      webxr="requiredFeatures:local-floor,hit-test,anchors;"
      geometry="primitive: plane"
    >
      <a-assets>
        <a-asset-item
          id="reticle"
          src="https://immersive-web.github.io/webxr-samples/media/gltf/reticle/reticle.gltf"
        ></a-asset-item>
      </a-assets>
      <a-entity
        material="primitive:plane;"
        visible="true"
        gltf-model="url(https://immersive-web.github.io/webxr-samples/media/gltf/reticle/reticle.gltf)"
        hit-test
      ></a-entity>
    </a-scene>
  </body>
</html>

The text was updated successfully, but these errors were encountered:

AdaRoseCannon · 2022-10-06T10:16:46Z

This looks like the expected behaviour, the system seemed to update the anchor position to restore to the correct position at the end.

Just so you know aframe already has a built in hit-test component: https://aframe.io/docs/1.3.0/components/ar-hit-test.html

latifs · 2022-10-06T16:03:10Z

Hi @AdaRoseCannon,
wouldn't you want the red and blue boxes to be inside the constraints of the tape on the ground at all times?
it is kind of disturbing that they are somehow closer to you at the start and then slowly drift into place as you get closer.

When I try the "same" experiment with the IKEA Place app and some random furniture, the drift is not noticeable.
So my thought was that I was doing something wrong with my position update.

Does it mean that if I use webXR and threeJS only, I'd see the same thing happen?

AdaRoseCannon · 2022-10-06T16:43:59Z

Were the red and blue box anchors created from hit test results?

latifs · 2022-10-06T16:56:23Z

The red and blue boxes are anchors but are created relative to the main anchor.
The main anchor was itself created by hit test results at the beginning of the video.

Once I place the main anchor I use its anchorPose and XRRigidTransform to place the other anchors at a certain distance from it.

The main idea with this is to try to spin up a world by placing objects (obstacles) relative to the main anchor.
These objects could be existing in the real world which is why accurate placement at all times is important.

AdaRoseCannon · 2022-10-06T17:17:28Z

Right that's your issue, the system's understanding of the scene itself doesn't maintain consistent size as it learns more about the environment so you need multiple anchors so the system can track those points as the system evolves.

bialpio · 2022-10-06T17:30:56Z

One more thing to try: before placing main anchor, can you walk around to the fixture markers and back? I wonder if the system didn't have enough data at the time you created the fixture anchors (fixtures seem to be rendered as if they are below ground, but w/o depth-based occlusion w/ the real world it's hard to tell).

latifs · 2022-10-06T17:45:46Z

@AdaRoseCannon Do you mean multiple anchors from hit test results?

Because I thought of that and

Option1:
one idea was to position real life anchors (tape on the ground) at each corner of the room and hit test results them to compute a master anchor position to use for placement of the boxes.
These 4 real life anchors would map the room to help better positioning. (probably overkill).

Option2:
Walk around the room (as @bialpio suggested) and every time you get a hit-test-result object just place anchors there to again map out the room better. Then when you are ready to place your main anchor (by clicking the screen) you have a better understanding of the room. (which I suspect is what IKEA place is doing).

Thanks for helping guys, much appreciated.

bialpio · 2022-10-06T17:54:41Z

Option2: Walk around the room (as @bialpio suggested) and every time you get a hit-test-result object just place anchors there to again map the room better.

I think you can try the existing code w/o any changes - I'm mostly curious if the current issue is because when you are placing the fixture anchors, the part of the environment that they are supposed to be in is not yet very well understood by the underlying system. I re-watched the video, and yup, all the anchors seem to initially be below the ground - main anchor gets fixed up at the end of the video, but others aren't (since they were not created based off of a hit-test result so they don't know they should "track" the ground).

AdaRoseCannon · 2022-10-06T18:21:59Z

Thanks @bialpio for putting it so well. I am on my phone.

klausw · 2022-10-06T18:50:53Z

In short, using a single anchor to place objects throughout a large space would only work if the AR system has a rigid global coordinate system, but that's not how ARCore (or other similar tracking systems) work.

Think of the AR system's world model as being made of rubber, and that model gets deformed and stretched during a session as it receives additional data and improves its understanding. Anchors based on hit test results correspond to pins placed where the hit test happened. These points will move along with their local piece of the rubber model. That's why it's important to use anchors close to where you are placing augmented objects. If you use a distant anchor to place an object, imagine that the object is connected to that anchor's pin with a long stick, and that will obviously wobble around as the rubber model deforms.

I guess this analogy is a bit of a stretch (pun intended), but I hope it helps build a better mental model of what's happening.

latifs · 2022-10-06T19:45:07Z

Hey Guys,

this is another attempt at trying to map the space better.
From what you guys were saying an understanding of the space is important in order to get the most accurate placement of the boxes.
I keep referencing the IKEA Place app (I don't work there, I promise) because it doesn't require you to walk around, scan areas of the room, or anything like that.
So my guess was that in the background some Anchors were put on the ground to map the space better and improve the future positioning of furniture pieces.

This is what I try to do in the video below (all the way down). I drop 400 anchors as I get a HitTestResults object (to map the space better).
And then go back to my main anchor position and hit the screen to get its placement.

Then I use this piece of code to place my boxes relative to the main anchor position.
Basically, the main anchor becomes my origin and I just add to the position of the fixture to the position of the main anchor (using XRRigidTransform).

createOffsetAnchor: async function (frame, fixtureData) {
  const { fx, fy, fz } = fixtureData.origin;
  const { x: ax, y: ay, z: az } = this.mainAnchorPose.position;
  const { x, y, z } = { x: ax + fx, y: ay, z: az + fz };

  let anchorFixture = await frame.createAnchor(
    new XRRigidTransform({ x, y, z }, { x: 0, y: 0, z: 0, w: 1 }),
    this.refSpace
  );
  
  this.anchoredObjects.push({
    id: fixtureData.id,
    anchor: anchorFixture,
  });
}

so either:

XRRigidTransform is doing something funky in the background. (it could also not be the correct way to place my boxes
I need to find a way for anchors not created from a hitTestResults to track the ground better as @bialpio hinted at (right now they are placed at the level the main anchor was placed at on click of the screen, see code above)
Create multiple master anchors and place the fixtures using the closest hit test results placed master anchors (as @klausw
suggested)

I'm willing to try other options.
Thanks all for the tips and the knowledge drops

Screen_Recording_20221006-120959_Chrome.mp4

Provide feedback

Saved searches

Use saved searches to filter your results more quickly

Anchors Drifting #157

Anchors Drifting #157

latifs commented Oct 5, 2022

AdaRoseCannon commented Oct 6, 2022

latifs commented Oct 6, 2022 •

edited

Loading

AdaRoseCannon commented Oct 6, 2022

latifs commented Oct 6, 2022 •

edited

Loading

AdaRoseCannon commented Oct 6, 2022

bialpio commented Oct 6, 2022

latifs commented Oct 6, 2022 •

edited

Loading

bialpio commented Oct 6, 2022

AdaRoseCannon commented Oct 6, 2022

klausw commented Oct 6, 2022

latifs commented Oct 6, 2022 •

edited

Loading

Anchors Drifting #157

Anchors Drifting #157

Comments

latifs commented Oct 5, 2022

AdaRoseCannon commented Oct 6, 2022

latifs commented Oct 6, 2022 • edited Loading

AdaRoseCannon commented Oct 6, 2022

latifs commented Oct 6, 2022 • edited Loading

AdaRoseCannon commented Oct 6, 2022

bialpio commented Oct 6, 2022

latifs commented Oct 6, 2022 • edited Loading

bialpio commented Oct 6, 2022

AdaRoseCannon commented Oct 6, 2022

klausw commented Oct 6, 2022

latifs commented Oct 6, 2022 • edited Loading

latifs commented Oct 6, 2022 •

edited

Loading

latifs commented Oct 6, 2022 •

edited

Loading

latifs commented Oct 6, 2022 •

edited

Loading

latifs commented Oct 6, 2022 •

edited

Loading