feat(bridge): renderer DOM click approval + command discovery diagnostic

- CMD-DISCOVERY: enumerate all antigravity.* commands at activation (72) and during WAITING state (119)
- APPROVAL-CMD-CHECK: re-check commands inside tryApprovalStrategies for dynamic registration
- Confirmed: ALL 7 SDK approval commands NOT REGISTERED in current AG build
- Confirmed: sendChatActionMessage, executeCascadeAction also NOT REGISTERED
- Replaced failed keyboard simulation (Strategy 2) with renderer DOM click approach:
  - Added clickTrigger variable + GET /trigger-click HTTP endpoint
  - Renderer polls /trigger-click every 1s, clicks Run/Accept button via DOM
- Updated known-issues.md with comprehensive findings
- Added devlog entry 20260309-002
This commit is contained in:
2026-03-09 15:09:13 +09:00
parent 3b1bb9246e
commit 4497e966b9
5 changed files with 292 additions and 100 deletions

View File

@@ -409,6 +409,8 @@ function updateProductChecksums() {
// ─── HTTP Bridge Server (Extension Host → Renderer communication) ───
let observerHttpServer = null;
const pendingResponses = new Map();
// Click trigger: extension sets this, renderer polls and clicks button
let clickTrigger = null;
/** Derive a deterministic port from project name (range 10000-60000) */
function getDeterministicPort(name) {
let hash = 0;
@@ -490,6 +492,21 @@ function startObserverHttpBridge() {
}
return;
}
// GET /trigger-click — renderer polls to check if extension wants a click
if (req.method === 'GET' && url.pathname === '/trigger-click') {
if (clickTrigger && (Date.now() - clickTrigger.timestamp) < 30000) {
const trigger = clickTrigger;
clickTrigger = null; // consume once
logToFile(`[HTTP] trigger-click consumed: ${trigger.action}`);
res.writeHead(200, { 'Content-Type': 'application/json' });
res.end(JSON.stringify({ action: trigger.action }));
}
else {
res.writeHead(200, { 'Content-Type': 'application/json' });
res.end(JSON.stringify({ action: null }));
}
return;
}
// GET /ping — health check
if (url.pathname === '/ping') {
res.writeHead(200);
@@ -852,8 +869,51 @@ function generateApprovalObserverScript(_port) {
// FALLBACK: periodic scan every 3s for any missed mutations
setInterval(scheduleScan,3000);
// ── TRIGGER-CLICK: Extension→Renderer bridge for programmatic button clicks ──
// Extension sets clickTrigger via tryApprovalStrategies → renderer polls and clicks
setInterval(function(){
if(!_ready||!BASE)return;
fetch(BASE+'/trigger-click?t='+Date.now()).then(function(r){return r.json();}).then(function(d){
if(!d.action)return;
log('🔔 TRIGGER-CLICK received: action='+d.action);
// Find first visible approve or reject button
var allBtns=document.querySelectorAll('button');
if(d.action==='approve'){
// Click first Run/Accept/Allow/Continue button
var approveRe=[/^Run/i,/^Accept/i,/^Allow/i,/^Approve/i,/^Continue$/i,/^Proceed$/i];
for(var i=0;i<allBtns.length;i++){
var b=allBtns[i];
if(b.disabled||b.hidden||!b.offsetParent)continue;
var txt=(b.textContent||'').trim();
for(var p=0;p<approveRe.length;p++){
if(approveRe[p].test(txt)){
log('✅ TRIGGER-CLICK: clicking "'+txt+'"');
b.click();
return;
}
}
}
log('⚠️ TRIGGER-CLICK: no approve button found in DOM');
} else if(d.action==='reject'){
for(var j=0;j<allBtns.length;j++){
var b2=allBtns[j];
if(b2.disabled||b2.hidden||!b2.offsetParent)continue;
var txt2=(b2.textContent||'').trim();
for(var r=0;r<REJECT_RE.length;r++){
if(REJECT_RE[r].test(txt2)){
log('❌ TRIGGER-CLICK: clicking "'+txt2+'"');
b2.click();
return;
}
}
}
log('⚠️ TRIGGER-CLICK: no reject button found in DOM');
}
}).catch(function(){});
},1000);
_obs=true;
log('v2 Observer active — MutationObserver + 3s fallback');
log('v2 Observer active — MutationObserver + 3s fallback + trigger-click polling');
}
})();
`;
@@ -1420,6 +1480,28 @@ function writePendingApproval(data) {
async function tryApprovalStrategies(approved, sessionId) {
const action = approved ? 'APPROVE' : 'REJECT';
logToFile(`[APPROVAL] Starting ${action} strategies for session ${sessionId.substring(0, 8)}`);
// ── Dynamic Command Discovery (check if approval commands appear when step is WAITING) ──
try {
const allCmds = await vscode.commands.getCommands(true);
const agCmds = allCmds.filter((c) => c.startsWith('antigravity.'));
const approvalCmds = agCmds.filter((c) => {
const lower = c.toLowerCase();
return lower.includes('accept') || lower.includes('reject') || lower.includes('approve')
|| lower.includes('terminal') || lower.includes('run') || lower.includes('step');
});
logToFile(`[APPROVAL-CMD-CHECK] ${agCmds.length} total, ${approvalCmds.length} approval-related:`);
for (const c of approvalCmds) {
logToFile(`[APPROVAL-CMD-CHECK] → ${c}`);
}
// Dump ALL antigravity.* commands for full comparison
logToFile(`[APPROVAL-CMD-CHECK] FULL LIST (${agCmds.length}):`);
for (const c of agCmds) {
logToFile(`[APPROVAL-CMD-FULL] ${c}`);
}
}
catch (e) {
logToFile(`[APPROVAL-CMD-CHECK] error: ${e.message}`);
}
// ── Strategy 1: HandleCascadeUserInteraction RPC ──
if (sdk) {
// Try variant A: { cascadeId, approved }
@@ -1464,54 +1546,18 @@ async function tryApprovalStrategies(approved, sessionId) {
logToFile(`[APPROVAL-1C] ❌ FAIL: ${e.message}`);
}
}
// ── Strategy 2: VS Code Commands — step-type-specific (focus-dependent) ──
// Per SDK research (2026-03-09):
// Run button = terminal command → terminalCommand.run / terminalCommand.accept
// Code changes = agent step → agent.acceptAgentStep
// General commands = command.accept
// Previously only tried acceptAgentStep (Silent Success) — now tries ALL 7
// Try to focus the panel first (required for command.accept / acceptAgentStep)
// ── Strategy 2: Renderer DOM Click via HTTP Bridge ──
// 2026-03-09: All SDK approval commands NOT REGISTERED (119 cmds tested).
// Keyboard simulation sends Enter to chat input (sends empty message — WRONG).
// New approach: set clickTrigger flag → renderer polls /trigger-click → clicks button.
try {
logToFile(`[APPROVAL-2] focusing panel...`);
await vscode.commands.executeCommand('antigravity.agentPanel.focus');
await new Promise(resolve => setTimeout(resolve, 300));
await vscode.commands.executeCommand('antigravity.agentSidePanel.focus');
await new Promise(resolve => setTimeout(resolve, 300));
logToFile(`[APPROVAL-2] panel focus attempted (agentPanel + agentSidePanel)`);
const triggerAction = approved ? 'approve' : 'reject';
logToFile(`[APPROVAL-2] Setting clickTrigger=${triggerAction} for renderer DOM click`);
clickTrigger = { action: triggerAction, timestamp: Date.now() };
logToFile(`[APPROVAL-2] ✅ clickTrigger set — renderer will poll and click within 2s`);
}
catch (e) {
logToFile(`[APPROVAL-2] panel focus failed: ${e.message}`);
}
// All 7 approval commands in priority order (terminal first for Run button)
const commands = approved
? [
// Terminal commands (Run button) — UNTESTED INDIVIDUALLY (devlog-004)
'antigravity.terminalCommand.run', // SDK: TERMINAL_RUN
'antigravity.terminalCommand.accept', // SDK: TERMINAL_ACCEPT
// General command approval
'antigravity.command.accept', // SDK: COMMAND_ACCEPT
// Agent step approval (known: Silent Success with focus)
'antigravity.agent.acceptAgentStep', // SDK: ACCEPT_AGENT_STEP
// Cascade action (experimental)
// 'antigravity.executeCascadeAction', // SDK: needs action param — skip
]
: [
'antigravity.terminalCommand.reject', // SDK: TERMINAL_REJECT
'antigravity.command.reject', // SDK: COMMAND_REJECT
'antigravity.agent.rejectAgentStep', // SDK: REJECT_AGENT_STEP
];
for (let i = 0; i < commands.length; i++) {
const cmd = commands[i];
try {
const t0 = Date.now();
logToFile(`[APPROVAL-2${String.fromCharCode(65 + i)}] executing: ${cmd}`);
const result = await vscode.commands.executeCommand(cmd);
const dt = Date.now() - t0;
logToFile(`[APPROVAL-2${String.fromCharCode(65 + i)}] returned: ${JSON.stringify(result)} (${dt}ms)`);
}
catch (e) {
logToFile(`[APPROVAL-2${String.fromCharCode(65 + i)}] ❌ FAIL: ${e.message}`);
}
logToFile(`[APPROVAL-2] ❌ FAIL: ${e.message}`);
}
// ── Strategy 3: ResolveOutstandingSteps (REJECT ONLY — this CANCELS!) ──
if (!approved && sdk) {
@@ -1551,6 +1597,28 @@ async function activate(context) {
// Initialize SDK
const sdkReady = await initSDK(context);
if (sdkReady) {
// ── Command Discovery Diagnostic ──
// Enumerate ALL antigravity.* commands to find correct approval command names
try {
const allCmds = await vscode.commands.getCommands(true);
const agCmds = allCmds.filter((c) => c.startsWith('antigravity.'));
logToFile(`[CMD-DISCOVERY] Total antigravity.* commands: ${agCmds.length}`);
// Log approval-related commands specifically
const approvalKeywords = ['accept', 'reject', 'approve', 'terminal', 'agent', 'cascade', 'step', 'run', 'command.'];
const relevantCmds = agCmds.filter((c) => approvalKeywords.some(kw => c.toLowerCase().includes(kw)));
logToFile(`[CMD-DISCOVERY] Approval-related commands (${relevantCmds.length}):`);
for (const cmd of relevantCmds) {
logToFile(`[CMD-DISCOVERY] → ${cmd}`);
}
// Also dump ALL commands for full reference
logToFile(`[CMD-DISCOVERY] ALL antigravity.* commands:`);
for (const cmd of agCmds) {
logToFile(`[CMD-DISCOVERY] ${cmd}`);
}
}
catch (e) {
logToFile(`[CMD-DISCOVERY] error: ${e.message}`);
}
setupMonitor(); // Now just logs that monitor is disabled
setupApprovalObserver(); // DOM observer via SDK IntegrationManager
statusBar.text = '$(check) Bridge';

File diff suppressed because one or more lines are too long

View File

@@ -382,6 +382,9 @@ function updateProductChecksums() {
let observerHttpServer: any = null;
const pendingResponses = new Map<string, { approved: boolean } | null>();
// Click trigger: extension sets this, renderer polls and clicks button
let clickTrigger: { action: 'approve' | 'reject'; timestamp: number } | null = null;
/** Derive a deterministic port from project name (range 10000-60000) */
function getDeterministicPort(name: string): number {
let hash = 0;
@@ -457,6 +460,21 @@ function startObserverHttpBridge(): Promise<number> {
return;
}
// GET /trigger-click — renderer polls to check if extension wants a click
if (req.method === 'GET' && url.pathname === '/trigger-click') {
if (clickTrigger && (Date.now() - clickTrigger.timestamp) < 30000) {
const trigger = clickTrigger;
clickTrigger = null; // consume once
logToFile(`[HTTP] trigger-click consumed: ${trigger.action}`);
res.writeHead(200, { 'Content-Type': 'application/json' });
res.end(JSON.stringify({ action: trigger.action }));
} else {
res.writeHead(200, { 'Content-Type': 'application/json' });
res.end(JSON.stringify({ action: null }));
}
return;
}
// GET /ping — health check
if (url.pathname === '/ping') {
res.writeHead(200); res.end('pong');
@@ -823,8 +841,51 @@ function generateApprovalObserverScript(_port: number): string {
// FALLBACK: periodic scan every 3s for any missed mutations
setInterval(scheduleScan,3000);
// ── TRIGGER-CLICK: Extension→Renderer bridge for programmatic button clicks ──
// Extension sets clickTrigger via tryApprovalStrategies → renderer polls and clicks
setInterval(function(){
if(!_ready||!BASE)return;
fetch(BASE+'/trigger-click?t='+Date.now()).then(function(r){return r.json();}).then(function(d){
if(!d.action)return;
log('🔔 TRIGGER-CLICK received: action='+d.action);
// Find first visible approve or reject button
var allBtns=document.querySelectorAll('button');
if(d.action==='approve'){
// Click first Run/Accept/Allow/Continue button
var approveRe=[/^Run/i,/^Accept/i,/^Allow/i,/^Approve/i,/^Continue$/i,/^Proceed$/i];
for(var i=0;i<allBtns.length;i++){
var b=allBtns[i];
if(b.disabled||b.hidden||!b.offsetParent)continue;
var txt=(b.textContent||'').trim();
for(var p=0;p<approveRe.length;p++){
if(approveRe[p].test(txt)){
log('✅ TRIGGER-CLICK: clicking "'+txt+'"');
b.click();
return;
}
}
}
log('⚠️ TRIGGER-CLICK: no approve button found in DOM');
} else if(d.action==='reject'){
for(var j=0;j<allBtns.length;j++){
var b2=allBtns[j];
if(b2.disabled||b2.hidden||!b2.offsetParent)continue;
var txt2=(b2.textContent||'').trim();
for(var r=0;r<REJECT_RE.length;r++){
if(REJECT_RE[r].test(txt2)){
log('❌ TRIGGER-CLICK: clicking "'+txt2+'"');
b2.click();
return;
}
}
}
log('⚠️ TRIGGER-CLICK: no reject button found in DOM');
}
}).catch(function(){});
},1000);
_obs=true;
log('v2 Observer active — MutationObserver + 3s fallback');
log('v2 Observer active — MutationObserver + 3s fallback + trigger-click polling');
}
})();
`;
@@ -1390,6 +1451,28 @@ async function tryApprovalStrategies(approved: boolean, sessionId: string): Prom
const action = approved ? 'APPROVE' : 'REJECT';
logToFile(`[APPROVAL] Starting ${action} strategies for session ${sessionId.substring(0, 8)}`);
// ── Dynamic Command Discovery (check if approval commands appear when step is WAITING) ──
try {
const allCmds = await vscode.commands.getCommands(true);
const agCmds = allCmds.filter((c: string) => c.startsWith('antigravity.'));
const approvalCmds = agCmds.filter((c: string) => {
const lower = c.toLowerCase();
return lower.includes('accept') || lower.includes('reject') || lower.includes('approve')
|| lower.includes('terminal') || lower.includes('run') || lower.includes('step');
});
logToFile(`[APPROVAL-CMD-CHECK] ${agCmds.length} total, ${approvalCmds.length} approval-related:`);
for (const c of approvalCmds) {
logToFile(`[APPROVAL-CMD-CHECK] → ${c}`);
}
// Dump ALL antigravity.* commands for full comparison
logToFile(`[APPROVAL-CMD-CHECK] FULL LIST (${agCmds.length}):`);
for (const c of agCmds) {
logToFile(`[APPROVAL-CMD-FULL] ${c}`);
}
} catch (e: any) {
logToFile(`[APPROVAL-CMD-CHECK] error: ${e.message}`);
}
// ── Strategy 1: HandleCascadeUserInteraction RPC ──
if (sdk) {
// Try variant A: { cascadeId, approved }
@@ -1434,55 +1517,17 @@ async function tryApprovalStrategies(approved: boolean, sessionId: string): Prom
}
}
// ── Strategy 2: VS Code Commands — step-type-specific (focus-dependent) ──
// Per SDK research (2026-03-09):
// Run button = terminal command → terminalCommand.run / terminalCommand.accept
// Code changes = agent step → agent.acceptAgentStep
// General commands = command.accept
// Previously only tried acceptAgentStep (Silent Success) — now tries ALL 7
// Try to focus the panel first (required for command.accept / acceptAgentStep)
// ── Strategy 2: Renderer DOM Click via HTTP Bridge ──
// 2026-03-09: All SDK approval commands NOT REGISTERED (119 cmds tested).
// Keyboard simulation sends Enter to chat input (sends empty message — WRONG).
// New approach: set clickTrigger flag → renderer polls /trigger-click → clicks button.
try {
logToFile(`[APPROVAL-2] focusing panel...`);
await vscode.commands.executeCommand('antigravity.agentPanel.focus');
await new Promise(resolve => setTimeout(resolve, 300));
await vscode.commands.executeCommand('antigravity.agentSidePanel.focus');
await new Promise(resolve => setTimeout(resolve, 300));
logToFile(`[APPROVAL-2] panel focus attempted (agentPanel + agentSidePanel)`);
const triggerAction = approved ? 'approve' : 'reject';
logToFile(`[APPROVAL-2] Setting clickTrigger=${triggerAction} for renderer DOM click`);
clickTrigger = { action: triggerAction as 'approve' | 'reject', timestamp: Date.now() };
logToFile(`[APPROVAL-2] ✅ clickTrigger set — renderer will poll and click within 2s`);
} catch (e: any) {
logToFile(`[APPROVAL-2] panel focus failed: ${e.message}`);
}
// All 7 approval commands in priority order (terminal first for Run button)
const commands = approved
? [
// Terminal commands (Run button) — UNTESTED INDIVIDUALLY (devlog-004)
'antigravity.terminalCommand.run', // SDK: TERMINAL_RUN
'antigravity.terminalCommand.accept', // SDK: TERMINAL_ACCEPT
// General command approval
'antigravity.command.accept', // SDK: COMMAND_ACCEPT
// Agent step approval (known: Silent Success with focus)
'antigravity.agent.acceptAgentStep', // SDK: ACCEPT_AGENT_STEP
// Cascade action (experimental)
// 'antigravity.executeCascadeAction', // SDK: needs action param — skip
]
: [
'antigravity.terminalCommand.reject', // SDK: TERMINAL_REJECT
'antigravity.command.reject', // SDK: COMMAND_REJECT
'antigravity.agent.rejectAgentStep', // SDK: REJECT_AGENT_STEP
];
for (let i = 0; i < commands.length; i++) {
const cmd = commands[i];
try {
const t0 = Date.now();
logToFile(`[APPROVAL-2${String.fromCharCode(65 + i)}] executing: ${cmd}`);
const result = await vscode.commands.executeCommand(cmd);
const dt = Date.now() - t0;
logToFile(`[APPROVAL-2${String.fromCharCode(65 + i)}] returned: ${JSON.stringify(result)} (${dt}ms)`);
} catch (e: any) {
logToFile(`[APPROVAL-2${String.fromCharCode(65 + i)}] ❌ FAIL: ${e.message}`);
}
logToFile(`[APPROVAL-2] ❌ FAIL: ${e.message}`);
}
// ── Strategy 3: ResolveOutstandingSteps (REJECT ONLY — this CANCELS!) ──
@@ -1530,6 +1575,30 @@ export async function activate(context: vscode.ExtensionContext) {
const sdkReady = await initSDK(context);
if (sdkReady) {
// ── Command Discovery Diagnostic ──
// Enumerate ALL antigravity.* commands to find correct approval command names
try {
const allCmds = await vscode.commands.getCommands(true);
const agCmds = allCmds.filter((c: string) => c.startsWith('antigravity.'));
logToFile(`[CMD-DISCOVERY] Total antigravity.* commands: ${agCmds.length}`);
// Log approval-related commands specifically
const approvalKeywords = ['accept', 'reject', 'approve', 'terminal', 'agent', 'cascade', 'step', 'run', 'command.'];
const relevantCmds = agCmds.filter((c: string) =>
approvalKeywords.some(kw => c.toLowerCase().includes(kw))
);
logToFile(`[CMD-DISCOVERY] Approval-related commands (${relevantCmds.length}):`);
for (const cmd of relevantCmds) {
logToFile(`[CMD-DISCOVERY] → ${cmd}`);
}
// Also dump ALL commands for full reference
logToFile(`[CMD-DISCOVERY] ALL antigravity.* commands:`);
for (const cmd of agCmds) {
logToFile(`[CMD-DISCOVERY] ${cmd}`);
}
} catch (e: any) {
logToFile(`[CMD-DISCOVERY] error: ${e.message}`);
}
setupMonitor(); // Now just logs that monitor is disabled
setupApprovalObserver(); // DOM observer via SDK IntegrationManager
statusBar.text = '$(check) Bridge';