Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
57 commits
Select commit Hold shift + click to select a range
1183b82
Remove Export to Local
MMMMMaasd Apr 11, 2025
6ab64d2
Audio enable option in setting
MMMMMaasd Apr 11, 2025
a915f2f
Remove Wifi in streaming option
MMMMMaasd Apr 11, 2025
8e22672
Incorporate Bluetooth Manager with AR Manager
MMMMMaasd Apr 11, 2025
16a69b6
Demos Counter
MMMMMaasd Apr 11, 2025
d37da15
New design of record button, fix demo counter bug
MMMMMaasd Apr 11, 2025
507d2b6
Added example model inferencing on-device
DjKesu May 30, 2025
9fb97cc
ML Inference with GeneralPickUp Policy
DjKesu Jun 1, 2025
b644900
display joint action values
DjKesu Jun 1, 2025
7dc0026
show joint action prediction values
DjKesu Jun 1, 2025
79af3ec
added visualization and model upload with management
DjKesu Jun 18, 2025
719ff92
Added visualization and fixed inference
DjKesu Jul 16, 2025
2d7aba6
Merge branch 'Krish/develop' of github.com:NYU-robot-learning/AnySens…
DjKesu Jul 16, 2025
5c261be
added edgetam support and prompt update logic
DjKesu Jul 30, 2025
466d408
Align USB robot action transform with server (P+Z90), unify VQ-BeT 3D…
DjKesu Aug 31, 2025
c9e91ce
Remove EdgeTAM feature files and references from branch; keep ML-only…
DjKesu Aug 31, 2025
8a0abc1
Viz: remove 0.5m/world-origin anchor; keep camera-relative arrows only
DjKesu Aug 31, 2025
e3628b4
Restore OdometryPointTracker.swift from history; reinstate odometry g…
DjKesu Aug 31, 2025
f76d859
Viz: restore target/device pose handlers (updateActualDevicePose/upda…
DjKesu Aug 31, 2025
32ff073
Unify policy→camera transforms; orientation-aware viz; ML USB mapping…
DjKesu Sep 1, 2025
6ef1147
removed models directory from pbxproj data
DjKesu Sep 1, 2025
def8da8
updated gitignore
DjKesu Sep 1, 2025
8433a14
added an iphone transform which was missing before, and fixed mobile …
DjKesu Sep 2, 2025
eeec973
fixed ux issues
DjKesu Sep 3, 2025
8f4c93f
removed differences in usb streaming mode on/off
DjKesu Sep 4, 2025
376a74c
fixed joint streaming
DjKesu Sep 7, 2025
129abe5
added anchoring method and simplified visualization
DjKesu Sep 9, 2025
d6dedc8
fixed coordinate issues
DjKesu Sep 9, 2025
b53a4e1
AR goal viz: revert to world-origin sphere flow; only visualize on go…
DjKesu Sep 10, 2025
2346ac9
fixed vis issue and reset goal point
DjKesu Sep 10, 2025
2da378c
non-point condition to 224, remove extra orentation down check
DjKesu Sep 10, 2025
e32bd4d
added depth as primary on lidar available phones
DjKesu Sep 11, 2025
45b1896
fixed goal transform, and working on viz
DjKesu Sep 13, 2025
d8faa6a
added image overlays, accelerate optimizations, and visualization fixes
DjKesu Sep 24, 2025
41abd7a
added gripper closed overlay and inference status with viz fixed
DjKesu Sep 24, 2025
417fa92
gripper overlay and visualization with new models
DjKesu Oct 15, 2025
5ffdddf
added reticle overlay
DjKesu Oct 16, 2025
5eb68b1
working on visualization
DjKesu Oct 28, 2025
2bad585
fixing goal point visualization
DjKesu Oct 28, 2025
0cbdc12
refactor depth to work on pro phones
KevinyWu Oct 29, 2025
adb8aff
fixed inferencing and visualization to cube/goal hud-style layout
DjKesu Nov 6, 2025
c2ca9cb
fixed gripper overlay
DjKesu Nov 12, 2025
961bb82
fixes and app launch beta
DjKesu Nov 18, 2025
f9798a8
policy transform updates
DjKesu Nov 21, 2025
16ba35b
more fixes to transform issues
DjKesu Nov 21, 2025
5d2800a
refactor redundant code, fix label and camera coordinates
DjKesu Nov 23, 2025
c35571a
fixed frame buffering
DjKesu Nov 26, 2025
bebcd05
usb streaming with no visuals
DjKesu Nov 27, 2025
f07f62b
fixed layout issues, updated managers, and ignored large mlpackage
DjKesu Nov 28, 2025
295007b
optimizing checkpoints
DjKesu Dec 27, 2025
d8eaae7
Prod changes and optimizations
DjKesu Dec 27, 2025
e0744f5
Final complete information including coreml conversion'
DjKesu Dec 27, 2025
d7af52a
final release version
DjKesu Dec 27, 2025
58b5722
added information on how to stream actions from iphone to robot
DjKesu Dec 27, 2025
66bef3e
Update inference UX and onboarding
DjKesu Jan 21, 2026
1287897
Remove welcome modal and default to inference
DjKesu Jan 22, 2026
b1fa12e
Updated readme
DjKesu Feb 9, 2026
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 7 additions & 0 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -64,3 +64,10 @@ fastlane/report.xml
fastlane/Preview.html
fastlane/screenshots/**/*.png
fastlane/test_output


# Models
Models/
*.mlpackage
AnySense.xcodeproj/project.xcworkspace/xcshareddata/swiftpm/
AnySense.xcworkspace/xcshareddata/swiftpm/
56 changes: 44 additions & 12 deletions AnySense.xcodeproj/project.pbxproj
Original file line number Diff line number Diff line change
Expand Up @@ -7,12 +7,20 @@
objects = {

/* Begin PBXBuildFile section */
016073392EF5BE610061563B /* InferenceView.swift in Sources */ = {isa = PBXBuildFile; fileRef = 016073382EF5BE610061563B /* InferenceView.swift */; };
01682BC42DF0D88100CDA03C /* ModelInfo.swift in Sources */ = {isa = PBXBuildFile; fileRef = 01F5B7C92DEA6A1000061B16 /* ModelInfo.swift */; };
01682BC52DF0D88C00CDA03C /* MLModel+Extensions.swift in Sources */ = {isa = PBXBuildFile; fileRef = 01F5B7CB2DEA6A2000061B16 /* MLModel+Extensions.swift */; };
01682BC92DF4D7CB00CDA03C /* ARVisualizationManager.swift in Sources */ = {isa = PBXBuildFile; fileRef = 01682BC82DF4D7CB00CDA03C /* ARVisualizationManager.swift */; };
01B77E6D2ED94950007823B6 /* demo.mlpackage in Resources */ = {isa = PBXBuildFile; fileRef = 01B77E6C2ED94950007823B6 /* demo.mlpackage */; };
01D2FDD82E64C33600F0BA98 /* ActionTransformUtils.swift in Sources */ = {isa = PBXBuildFile; fileRef = 01D2FDD52E64C33600F0BA98 /* ActionTransformUtils.swift */; };
01F5B7C42DEA697200061B16 /* MLInferenceManager.swift in Sources */ = {isa = PBXBuildFile; fileRef = 01F5B7C32DEA697200061B16 /* MLInferenceManager.swift */; };
01F5B7C62DEA698D00061B16 /* MLInferenceResultsView.swift in Sources */ = {isa = PBXBuildFile; fileRef = 01F5B7C52DEA698D00061B16 /* MLInferenceResultsView.swift */; };
01F5B7C82DEA6A0000061B16 /* ModelManager.swift in Sources */ = {isa = PBXBuildFile; fileRef = 01F5B7C72DEA6A0000061B16 /* ModelManager.swift */; };
B9104F622BFDE63C000D4DDD /* Assets.xcassets in Resources */ = {isa = PBXBuildFile; fileRef = B9104F612BFDE63C000D4DDD /* Assets.xcassets */; };
B9944A342CFE983300232FBB /* MainPage.swift in Sources */ = {isa = PBXBuildFile; fileRef = B9944A332CFE982E00232FBB /* MainPage.swift */; };
B997CCD32D4E081300F62B49 /* dataStorage.swift in Sources */ = {isa = PBXBuildFile; fileRef = B997CCD22D4E081300F62B49 /* dataStorage.swift */; };
B997CCD52D4E082900F62B49 /* AnySenseApp.swift in Sources */ = {isa = PBXBuildFile; fileRef = B997CCD42D4E082900F62B49 /* AnySenseApp.swift */; };
B9E4B22D2D62A9BA0032E877 /* BluetoothManager.swift in Sources */ = {isa = PBXBuildFile; fileRef = B9E4B2292D62A9BA0032E877 /* BluetoothManager.swift */; };
B9E4B22E2D62A9BA0032E877 /* WebRTCManager.swift in Sources */ = {isa = PBXBuildFile; fileRef = B9E4B22B2D62A9BA0032E877 /* WebRTCManager.swift */; };
B9E4B22F2D62A9BA0032E877 /* ARViewContainer.swift in Sources */ = {isa = PBXBuildFile; fileRef = B9E4B2282D62A9BA0032E877 /* ARViewContainer.swift */; };
B9E4B2302D62A9BA0032E877 /* USBManager.swift in Sources */ = {isa = PBXBuildFile; fileRef = B9E4B22A2D62A9BA0032E877 /* USBManager.swift */; };
B9E4B2362D62A9C40032E877 /* peripheralView.swift in Sources */ = {isa = PBXBuildFile; fileRef = B9E4B2332D62A9C40032E877 /* peripheralView.swift */; };
Expand Down Expand Up @@ -42,6 +50,15 @@
/* End PBXContainerItemProxy section */

/* Begin PBXFileReference section */
016073382EF5BE610061563B /* InferenceView.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = InferenceView.swift; sourceTree = "<group>"; };
01682BC82DF4D7CB00CDA03C /* ARVisualizationManager.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = ARVisualizationManager.swift; sourceTree = "<group>"; };
01B77E6C2ED94950007823B6 /* demo.mlpackage */ = {isa = PBXFileReference; lastKnownFileType = folder.mlpackage; path = demo.mlpackage; sourceTree = "<group>"; };
01D2FDD52E64C33600F0BA98 /* ActionTransformUtils.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = ActionTransformUtils.swift; sourceTree = "<group>"; };
01F5B7C32DEA697200061B16 /* MLInferenceManager.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = MLInferenceManager.swift; sourceTree = "<group>"; };
01F5B7C52DEA698D00061B16 /* MLInferenceResultsView.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = MLInferenceResultsView.swift; sourceTree = "<group>"; };
01F5B7C72DEA6A0000061B16 /* ModelManager.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = ModelManager.swift; sourceTree = "<group>"; };
01F5B7C92DEA6A1000061B16 /* ModelInfo.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = ModelInfo.swift; sourceTree = "<group>"; };
01F5B7CB2DEA6A2000061B16 /* MLModel+Extensions.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "MLModel+Extensions.swift"; sourceTree = "<group>"; };
B9104F5A2BFDE63A000D4DDD /* AnySense.app */ = {isa = PBXFileReference; explicitFileType = wrapper.application; includeInIndex = 0; path = AnySense.app; sourceTree = BUILT_PRODUCTS_DIR; };
B9104F612BFDE63C000D4DDD /* Assets.xcassets */ = {isa = PBXFileReference; lastKnownFileType = folder.assetcatalog; path = Assets.xcassets; sourceTree = "<group>"; };
B9104F6A2BFDE63C000D4DDD /* AnySenseTests.xctest */ = {isa = PBXFileReference; explicitFileType = wrapper.cfbundle; includeInIndex = 0; path = AnySenseTests.xctest; sourceTree = BUILT_PRODUCTS_DIR; };
Expand All @@ -53,7 +70,6 @@
B9E4B2282D62A9BA0032E877 /* ARViewContainer.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = ARViewContainer.swift; sourceTree = "<group>"; };
B9E4B2292D62A9BA0032E877 /* BluetoothManager.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = BluetoothManager.swift; sourceTree = "<group>"; };
B9E4B22A2D62A9BA0032E877 /* USBManager.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = USBManager.swift; sourceTree = "<group>"; };
B9E4B22B2D62A9BA0032E877 /* WebRTCManager.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = WebRTCManager.swift; sourceTree = "<group>"; };
B9E4B2312D62A9C40032E877 /* accountView.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = accountView.swift; sourceTree = "<group>"; };
B9E4B2322D62A9C40032E877 /* ContentView.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = ContentView.swift; sourceTree = "<group>"; };
B9E4B2332D62A9C40032E877 /* peripheralView.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = peripheralView.swift; sourceTree = "<group>"; };
Expand Down Expand Up @@ -110,6 +126,9 @@
B9104F5C2BFDE63A000D4DDD /* AnySense */ = {
isa = PBXGroup;
children = (
01F5B7C92DEA6A1000061B16 /* ModelInfo.swift */,
01B77E6C2ED94950007823B6 /* demo.mlpackage */,
01F5B7CB2DEA6A2000061B16 /* MLModel+Extensions.swift */,
B9944A332CFE982E00232FBB /* MainPage.swift */,
B997CCD22D4E081300F62B49 /* dataStorage.swift */,
B9104F612BFDE63C000D4DDD /* Assets.xcassets */,
Expand All @@ -126,18 +145,23 @@
isa = PBXGroup;
children = (
B9E4B2282D62A9BA0032E877 /* ARViewContainer.swift */,
01F5B7C32DEA697200061B16 /* MLInferenceManager.swift */,
01D2FDD52E64C33600F0BA98 /* ActionTransformUtils.swift */,
01682BC82DF4D7CB00CDA03C /* ARVisualizationManager.swift */,
01F5B7C72DEA6A0000061B16 /* ModelManager.swift */,
B9E4B2292D62A9BA0032E877 /* BluetoothManager.swift */,
B9E4B22A2D62A9BA0032E877 /* USBManager.swift */,
B9E4B22B2D62A9BA0032E877 /* WebRTCManager.swift */,
);
path = Managers;
sourceTree = "<group>";
};
B9E4B2352D62A9C40032E877 /* Views */ = {
isa = PBXGroup;
children = (
01F5B7C52DEA698D00061B16 /* MLInferenceResultsView.swift */,
B9E4B2312D62A9C40032E877 /* accountView.swift */,
B9E4B2322D62A9C40032E877 /* ContentView.swift */,
016073382EF5BE610061563B /* InferenceView.swift */,
B9E4B2332D62A9C40032E877 /* peripheralView.swift */,
B9E4B2342D62A9C40032E877 /* readView.swift */,
);
Expand Down Expand Up @@ -266,6 +290,7 @@
isa = PBXResourcesBuildPhase;
buildActionMask = 2147483647;
files = (
01B77E6D2ED94950007823B6 /* demo.mlpackage in Resources */,
B9104F622BFDE63C000D4DDD /* Assets.xcassets in Resources */,
B9E4D2B42D6965B90044F2D4 /* AnySenseLaunchScreen.storyboard in Resources */,
);
Expand Down Expand Up @@ -319,16 +344,23 @@
isa = PBXSourcesBuildPhase;
buildActionMask = 2147483647;
files = (
01682BC42DF0D88100CDA03C /* ModelInfo.swift in Sources */,
B9E4B2362D62A9C40032E877 /* peripheralView.swift in Sources */,
01682BC92DF4D7CB00CDA03C /* ARVisualizationManager.swift in Sources */,
B9E4B2372D62A9C40032E877 /* accountView.swift in Sources */,
B9E4B2382D62A9C40032E877 /* readView.swift in Sources */,
B9E4B2392D62A9C40032E877 /* ContentView.swift in Sources */,
B997CCD52D4E082900F62B49 /* AnySenseApp.swift in Sources */,
B997CCD32D4E081300F62B49 /* dataStorage.swift in Sources */,
01682BC52DF0D88C00CDA03C /* MLModel+Extensions.swift in Sources */,
01F5B7C42DEA697200061B16 /* MLInferenceManager.swift in Sources */,
01D2FDD82E64C33600F0BA98 /* ActionTransformUtils.swift in Sources */,
01F5B7C82DEA6A0000061B16 /* ModelManager.swift in Sources */,
016073392EF5BE610061563B /* InferenceView.swift in Sources */,
B9E4B22D2D62A9BA0032E877 /* BluetoothManager.swift in Sources */,
B9E4B22E2D62A9BA0032E877 /* WebRTCManager.swift in Sources */,
B9E4B22F2D62A9BA0032E877 /* ARViewContainer.swift in Sources */,
B9E4B2302D62A9BA0032E877 /* USBManager.swift in Sources */,
01F5B7C62DEA698D00061B16 /* MLInferenceResultsView.swift in Sources */,
B9944A342CFE983300232FBB /* MainPage.swift in Sources */,
);
runOnlyForDeploymentPostprocessing = 0;
Expand Down Expand Up @@ -492,7 +524,7 @@
ASSETCATALOG_COMPILER_GLOBAL_ACCENT_COLOR_NAME = AccentColor;
CODE_SIGN_IDENTITY = "Apple Development";
CODE_SIGN_STYLE = Automatic;
CURRENT_PROJECT_VERSION = 4;
CURRENT_PROJECT_VERSION = 21;
DEVELOPMENT_ASSET_PATHS = "";
DEVELOPMENT_TEAM = 88NB9U5CK6;
ENABLE_PREVIEWS = YES;
Expand All @@ -516,7 +548,7 @@
"$(inherited)",
"@executable_path/Frameworks",
);
MARKETING_VERSION = 1.0;
MARKETING_VERSION = 1.3;
PRODUCT_BUNDLE_IDENTIFIER = GRAIL.AnySense;
PRODUCT_NAME = "$(TARGET_NAME)";
PROVISIONING_PROFILE_SPECIFIER = "";
Expand All @@ -538,7 +570,7 @@
ASSETCATALOG_COMPILER_GLOBAL_ACCENT_COLOR_NAME = AccentColor;
CODE_SIGN_IDENTITY = "Apple Development";
CODE_SIGN_STYLE = Automatic;
CURRENT_PROJECT_VERSION = 4;
CURRENT_PROJECT_VERSION = 21;
DEVELOPMENT_ASSET_PATHS = "";
DEVELOPMENT_TEAM = 88NB9U5CK6;
ENABLE_PREVIEWS = YES;
Expand All @@ -561,7 +593,7 @@
"$(inherited)",
"@executable_path/Frameworks",
);
MARKETING_VERSION = 1.0;
MARKETING_VERSION = 1.3;
PRODUCT_BUNDLE_IDENTIFIER = GRAIL.AnySense;
PRODUCT_NAME = "$(TARGET_NAME)";
PROVISIONING_PROFILE_SPECIFIER = "";
Expand All @@ -581,7 +613,7 @@
BUNDLE_LOADER = "$(TEST_HOST)";
CODE_SIGN_STYLE = Automatic;
CURRENT_PROJECT_VERSION = 1;
DEVELOPMENT_TEAM = 9U23FMDF45;
DEVELOPMENT_TEAM = 57DF45Y5K2;
ENABLE_USER_SCRIPT_SANDBOXING = NO;
GENERATE_INFOPLIST_FILE = YES;
IPHONEOS_DEPLOYMENT_TARGET = 17.4;
Expand All @@ -601,7 +633,7 @@
BUNDLE_LOADER = "$(TEST_HOST)";
CODE_SIGN_STYLE = Automatic;
CURRENT_PROJECT_VERSION = 1;
DEVELOPMENT_TEAM = 9U23FMDF45;
DEVELOPMENT_TEAM = 57DF45Y5K2;
ENABLE_USER_SCRIPT_SANDBOXING = NO;
GENERATE_INFOPLIST_FILE = YES;
IPHONEOS_DEPLOYMENT_TARGET = 17.4;
Expand All @@ -620,7 +652,7 @@
buildSettings = {
CODE_SIGN_STYLE = Automatic;
CURRENT_PROJECT_VERSION = 1;
DEVELOPMENT_TEAM = 9U23FMDF45;
DEVELOPMENT_TEAM = 57DF45Y5K2;
ENABLE_USER_SCRIPT_SANDBOXING = NO;
GENERATE_INFOPLIST_FILE = YES;
MARKETING_VERSION = 1.0;
Expand All @@ -638,7 +670,7 @@
buildSettings = {
CODE_SIGN_STYLE = Automatic;
CURRENT_PROJECT_VERSION = 1;
DEVELOPMENT_TEAM = 9U23FMDF45;
DEVELOPMENT_TEAM = 57DF45Y5K2;
ENABLE_USER_SCRIPT_SANDBOXING = NO;
GENERATE_INFOPLIST_FILE = YES;
MARKETING_VERSION = 1.0;
Expand Down
8 changes: 8 additions & 0 deletions AnySense.xcworkspace/xcshareddata/IDEWorkspaceChecks.plist
Original file line number Diff line number Diff line change
@@ -0,0 +1,8 @@
<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
<plist version="1.0">
<dict>
<key>IDEDidComputeMac32BitWarning</key>
<true/>
</dict>
</plist>
1 change: 1 addition & 0 deletions AnySense/.gitignore
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
Models/
2 changes: 0 additions & 2 deletions AnySense/AnySenseApp.swift
Original file line number Diff line number Diff line change
Expand Up @@ -11,12 +11,10 @@ import BackgroundTasks
@main
struct AnySenseApp: App {
@StateObject var appStatus = AppInformation()
@StateObject var bluetoothManager = BluetoothManager()
var body: some Scene {
WindowGroup {
ContentView()
.environmentObject(appStatus)
.environmentObject(bluetoothManager)
}
}
}
21 changes: 21 additions & 0 deletions AnySense/Assets.xcassets/gripper_closed.imageset/Contents.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,21 @@
{
"images" : [
{
"filename" : "gripper_closed.png",
"idiom" : "universal",
"scale" : "1x"
},
{
"idiom" : "universal",
"scale" : "2x"
},
{
"idiom" : "universal",
"scale" : "3x"
}
],
"info" : {
"author" : "xcode",
"version" : 1
}
}
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
21 changes: 21 additions & 0 deletions AnySense/Assets.xcassets/gripper_overlay.imageset/Contents.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,21 @@
{
"images" : [
{
"filename" : "gripper_overlay.png",
"idiom" : "universal",
"scale" : "1x"
},
{
"idiom" : "universal",
"scale" : "2x"
},
{
"idiom" : "universal",
"scale" : "3x"
}
],
"info" : {
"author" : "xcode",
"version" : 1
}
}
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Loading