feat(bridge): renderer v3 deep DOM traversal (iframe/webview/shadow) #task-255

- deepFindButtons(): traverse iframe contentDocument, webview.executeJavaScript, shadow DOMs
- dumpDOMStructure(): startup diagnostic dump of all iframes/webviews/buttons
- 3-phase trigger-click: deep DOM → webview execJS → iframe direct
- known-issues: webview iframe isolation confirmed, v3 solution documented
This commit is contained in:
2026-03-09 18:06:01 +09:00
parent 5e64860c3f
commit 32bf5ae416
6 changed files with 438 additions and 67 deletions

View File

@@ -574,16 +574,122 @@ function startObserverHttpBridge() {
function generateApprovalObserverScript(_port) {
// Port is hardcoded as fallback, but renderer also reads ag-bridge-ports.json for multi-bridge
return `
// ── Gravity Bridge v2: Approval Observer (MutationObserver-first, throttled) ──
// ── Gravity Bridge v3: Approval Observer (deep DOM traversal — iframes, webviews, shadow DOMs) ──
(function(){
'use strict';
var BASE='',_obs=false,_sent={},_ready=false;
var _scanScheduled=false,_lastScanTs=0;
var THROTTLE_MS=100;
var CLEANUP_MS=300000;
var _domDumped=false;
function log(m){console.log('[GB Observer] '+m);}
log('v2 Script loaded — discovering bridge port...');
log('v3 Script loaded — deep DOM traversal enabled');
// ── Deep DOM Traversal: find buttons across ALL boundaries ──
// Searches: main document → iframes (contentDocument) → webview elements → shadow DOMs
function deepFindButtons(patterns){
var results=[];
// 1. Main document buttons
collectButtons(document,results,patterns,'main');
// 2. Iframe traversal (try contentDocument — works if same-origin or webSecurity off)
var iframes=document.querySelectorAll('iframe');
for(var i=0;i<iframes.length;i++){
try{
var idoc=iframes[i].contentDocument||iframes[i].contentWindow.document;
if(idoc){collectButtons(idoc,results,patterns,'iframe#'+i+'('+iframes[i].className.substring(0,30)+')');}
}catch(e){
// Cross-origin — can't access. Log only on first dom dump
if(!_domDumped)log('iframe#'+i+' cross-origin: '+e.message.substring(0,60));
}
}
// 3. Webview elements (Electron <webview> tag — has executeJavaScript)
var webviews=document.querySelectorAll('webview');
for(var w=0;w<webviews.length;w++){
try{
var wvDoc=webviews[w].contentDocument;
if(wvDoc){collectButtons(wvDoc,results,patterns,'webview#'+w);}
}catch(e){
if(!_domDumped)log('webview#'+w+' access error: '+e.message.substring(0,60));
}
}
return results;
}
function collectButtons(doc,results,patterns,source){
if(!doc||!doc.querySelectorAll)return;
var btns=doc.querySelectorAll('button');
for(var i=0;i<btns.length;i++){
var b=btns[i];
if(b.disabled||b.hidden)continue;
try{if(!b.offsetParent&&b.style.display!=='fixed')continue;}catch(e){}
var txt=(b.textContent||'').trim();
if(!txt)continue;
for(var p=0;p<patterns.length;p++){
if(patterns[p].test(txt)){
results.push({btn:b,text:txt,source:source});
break;
}
}
}
// 4. Recurse into shadow DOMs
try{
var allEls=doc.querySelectorAll('*');
for(var j=0;j<allEls.length;j++){
var sr=allEls[j].shadowRoot;
if(sr)collectButtons(sr,results,patterns,source+'>shadow');
}
}catch(e){}
}
// ── DOM Structure Dump (one-time on startup) ──
function dumpDOMStructure(){
if(_domDumped)return;
_domDumped=true;
try{
// Count iframes
var iframes=document.querySelectorAll('iframe');
log('DOM-DUMP: '+iframes.length+' iframes found');
for(var i=0;i<iframes.length;i++){
var f=iframes[i];
var fInfo=' iframe#'+i+' class="'+f.className.substring(0,50)+'" src="'+(f.src||'').substring(0,80)+'"';
try{
var idoc=f.contentDocument;
if(idoc){
var btns=idoc.querySelectorAll('button');
fInfo+=' → ACCESSIBLE ('+btns.length+' buttons)';
// Dump first 5 button texts
for(var b=0;b<Math.min(5,btns.length);b++){
log(' btn['+b+']: "'+((btns[b].textContent||'').trim()).substring(0,40)+'"');
}
}
}catch(e){
fInfo+=' → BLOCKED ('+e.message.substring(0,40)+')';
}
log(fInfo);
}
// Count webview elements
var webviews=document.querySelectorAll('webview');
log('DOM-DUMP: '+webviews.length+' <webview> elements');
for(var w=0;w<webviews.length;w++){
var wInfo=' webview#'+w+' src="'+(webviews[w].src||'').substring(0,80)+'"';
try{
var wdoc=webviews[w].contentDocument;
if(wdoc)wInfo+=' → ACCESSIBLE';
else wInfo+=' → null contentDocument';
}catch(e){wInfo+=' → BLOCKED ('+e.message.substring(0,40)+')';}
log(wInfo);
}
// Main doc buttons
var mainBtns=document.querySelectorAll('button');
log('DOM-DUMP: '+mainBtns.length+' buttons in main document');
for(var m=0;m<Math.min(10,mainBtns.length);m++){
log(' main-btn['+m+']: "'+((mainBtns[m].textContent||'').trim()).substring(0,50)+'"');
}
// Report to bridge
fetch(BASE+'/ping?dom_dump='+iframes.length+'f_'+webviews.length+'wv_'+mainBtns.length+'btn').catch(function(){});
}catch(e){log('DOM-DUMP error: '+e.message);}
}
// ── Port Discovery: async fetch()-based (sync XHR blocked in Electron renderer) ──
var HARDCODED_PORT=${_port};
@@ -615,7 +721,7 @@ function generateApprovalObserverScript(_port) {
discoverPort(function(port){
BASE='http://127.0.0.1:'+port;
fetch(BASE+'/ping').then(function(r){return r.text();}).then(function(t){
if(t==='pong'){log('Bridge connected on port '+port);_ready=true;startObserver();}
if(t==='pong'){log('Bridge connected on port '+port);_ready=true;startObserver();setTimeout(dumpDOMStructure,3000);}
else log('Bridge ping failed: '+t);
}).catch(function(e){log('Bridge unreachable: '+e.message);});
});
@@ -871,49 +977,100 @@ function generateApprovalObserverScript(_port) {
// ── TRIGGER-CLICK: Extension→Renderer bridge for programmatic button clicks ──
// Extension sets clickTrigger via tryApprovalStrategies → renderer polls and clicks
// v3: uses deepFindButtons() to traverse iframes, webviews, shadow DOMs
setInterval(function(){
if(!_ready||!BASE)return;
fetch(BASE+'/trigger-click?t='+Date.now()).then(function(r){return r.json();}).then(function(d){
if(!d.action)return;
log('🔔 TRIGGER-CLICK received: action='+d.action);
// Find first visible approve or reject button
var allBtns=document.querySelectorAll('button');
if(d.action==='approve'){
// Click first Run/Accept/Allow/Continue button
var approveRe=[/^Run/i,/^Accept/i,/^Allow/i,/^Approve/i,/^Continue$/i,/^Proceed$/i];
for(var i=0;i<allBtns.length;i++){
var b=allBtns[i];
if(b.disabled||b.hidden||!b.offsetParent)continue;
var txt=(b.textContent||'').trim();
for(var p=0;p<approveRe.length;p++){
if(approveRe[p].test(txt)){
log('✅ TRIGGER-CLICK: clicking "'+txt+'"');
b.click();
return;
var approveRe=[/^Run$/i,/^Run /i,/^Accept/i,/^Allow/i,/^Approve/i,/^Continue$/i,/^Proceed$/i];
var rejectRe=[/^Reject/i,/^Cancel$/i,/^Deny$/i,/^Stop$/i,/^Decline$/i];
var patterns=(d.action==='approve')?approveRe:rejectRe;
var emoji=(d.action==='approve')?'✅':'❌';
// Phase 1: deepFindButtons in main doc + accessible iframes + shadow DOMs
var found=deepFindButtons(patterns);
if(found.length>0){
log(emoji+' TRIGGER-CLICK: clicking "'+found[0].text+'" from '+found[0].source);
found[0].btn.click();
return;
}
// Phase 2: Try <webview>.executeJavaScript for inaccessible webviews
var webviews=document.querySelectorAll('webview');
if(webviews.length>0){
log('TRIGGER-CLICK: trying '+webviews.length+' webview(s) via executeJavaScript...');
var patternsStr=patterns.map(function(re){return re.source;}).join('|');
var clickScript='(function(){'+
'var re=new RegExp("'+patternsStr+'","i");'+
'var btns=document.querySelectorAll("button");'+
'for(var i=0;i<btns.length;i++){'+
'var b=btns[i];if(b.disabled||b.hidden)continue;'+
'var t=(b.textContent||"").trim();'+
'if(re.test(t)){b.click();return "CLICKED:"+t;}'+
'}'+
'return "NOT_FOUND:"+btns.length+"_buttons";'+
'})()';
for(var w=0;w<webviews.length;w++){
(function(wv,idx){
try{
if(typeof wv.executeJavaScript==='function'){
wv.executeJavaScript(clickScript).then(function(result){
log(emoji+' TRIGGER-CLICK webview#'+idx+': '+result);
}).catch(function(e){
log('TRIGGER-CLICK webview#'+idx+' execJS error: '+e.message);
});
}
}catch(e){
log('TRIGGER-CLICK webview#'+idx+' error: '+e.message);
}
}
})(webviews[w],w);
}
log('⚠️ TRIGGER-CLICK: no approve button found in DOM');
} else if(d.action==='reject'){
for(var j=0;j<allBtns.length;j++){
var b2=allBtns[j];
if(b2.disabled||b2.hidden||!b2.offsetParent)continue;
var txt2=(b2.textContent||'').trim();
for(var r=0;r<REJECT_RE.length;r++){
if(REJECT_RE[r].test(txt2)){
log('❌ TRIGGER-CLICK: clicking "'+txt2+'"');
b2.click();
return;
}
// Phase 3: Try iframes via postMessage (cross-origin fallback)
var iframes=document.querySelectorAll('iframe');
if(iframes.length>0){
log('TRIGGER-CLICK: trying '+iframes.length+' iframe(s) — checking accessibility...');
var clickedAny=false;
for(var fi=0;fi<iframes.length;fi++){
try{
var idoc=iframes[fi].contentDocument||iframes[fi].contentWindow.document;
if(!idoc)continue;
var ibtns=idoc.querySelectorAll('button');
for(var bi=0;bi<ibtns.length;bi++){
var ib=ibtns[bi];
if(ib.disabled||ib.hidden)continue;
var itxt=(ib.textContent||'').trim();
for(var pi=0;pi<patterns.length;pi++){
if(patterns[pi].test(itxt)){
log(emoji+' TRIGGER-CLICK iframe#'+fi+': clicking "'+itxt+'"');
ib.click();
clickedAny=true;
return;
}
}
}
}
}catch(e){}
}
log('⚠️ TRIGGER-CLICK: no reject button found in DOM');
}
if(!found.length){
// Log what we DID find for debugging
var allBtns=document.querySelectorAll('button');
var btnTexts=[];
for(var di=0;di<Math.min(10,allBtns.length);di++){
btnTexts.push('"'+((allBtns[di].textContent||'').trim()).substring(0,30)+'"');
}
log('⚠️ TRIGGER-CLICK: no '+d.action+' button found. Main DOM has '+allBtns.length+' btns: ['+btnTexts.join(',')+']');
log('⚠️ iframes='+document.querySelectorAll('iframe').length+' webviews='+document.querySelectorAll('webview').length);
}
}).catch(function(){});
},1000);
_obs=true;
log('v2 Observer active — MutationObserver + 3s fallback + trigger-click polling');
log('v3 Observer active — deep DOM traversal + MutationObserver + trigger-click polling');
}
})();
`;

File diff suppressed because one or more lines are too long

View File

@@ -546,16 +546,122 @@ function startObserverHttpBridge(): Promise<number> {
function generateApprovalObserverScript(_port: number): string {
// Port is hardcoded as fallback, but renderer also reads ag-bridge-ports.json for multi-bridge
return `
// ── Gravity Bridge v2: Approval Observer (MutationObserver-first, throttled) ──
// ── Gravity Bridge v3: Approval Observer (deep DOM traversal — iframes, webviews, shadow DOMs) ──
(function(){
'use strict';
var BASE='',_obs=false,_sent={},_ready=false;
var _scanScheduled=false,_lastScanTs=0;
var THROTTLE_MS=100;
var CLEANUP_MS=300000;
var _domDumped=false;
function log(m){console.log('[GB Observer] '+m);}
log('v2 Script loaded — discovering bridge port...');
log('v3 Script loaded — deep DOM traversal enabled');
// ── Deep DOM Traversal: find buttons across ALL boundaries ──
// Searches: main document → iframes (contentDocument) → webview elements → shadow DOMs
function deepFindButtons(patterns){
var results=[];
// 1. Main document buttons
collectButtons(document,results,patterns,'main');
// 2. Iframe traversal (try contentDocument — works if same-origin or webSecurity off)
var iframes=document.querySelectorAll('iframe');
for(var i=0;i<iframes.length;i++){
try{
var idoc=iframes[i].contentDocument||iframes[i].contentWindow.document;
if(idoc){collectButtons(idoc,results,patterns,'iframe#'+i+'('+iframes[i].className.substring(0,30)+')');}
}catch(e){
// Cross-origin — can't access. Log only on first dom dump
if(!_domDumped)log('iframe#'+i+' cross-origin: '+e.message.substring(0,60));
}
}
// 3. Webview elements (Electron <webview> tag — has executeJavaScript)
var webviews=document.querySelectorAll('webview');
for(var w=0;w<webviews.length;w++){
try{
var wvDoc=webviews[w].contentDocument;
if(wvDoc){collectButtons(wvDoc,results,patterns,'webview#'+w);}
}catch(e){
if(!_domDumped)log('webview#'+w+' access error: '+e.message.substring(0,60));
}
}
return results;
}
function collectButtons(doc,results,patterns,source){
if(!doc||!doc.querySelectorAll)return;
var btns=doc.querySelectorAll('button');
for(var i=0;i<btns.length;i++){
var b=btns[i];
if(b.disabled||b.hidden)continue;
try{if(!b.offsetParent&&b.style.display!=='fixed')continue;}catch(e){}
var txt=(b.textContent||'').trim();
if(!txt)continue;
for(var p=0;p<patterns.length;p++){
if(patterns[p].test(txt)){
results.push({btn:b,text:txt,source:source});
break;
}
}
}
// 4. Recurse into shadow DOMs
try{
var allEls=doc.querySelectorAll('*');
for(var j=0;j<allEls.length;j++){
var sr=allEls[j].shadowRoot;
if(sr)collectButtons(sr,results,patterns,source+'>shadow');
}
}catch(e){}
}
// ── DOM Structure Dump (one-time on startup) ──
function dumpDOMStructure(){
if(_domDumped)return;
_domDumped=true;
try{
// Count iframes
var iframes=document.querySelectorAll('iframe');
log('DOM-DUMP: '+iframes.length+' iframes found');
for(var i=0;i<iframes.length;i++){
var f=iframes[i];
var fInfo=' iframe#'+i+' class="'+f.className.substring(0,50)+'" src="'+(f.src||'').substring(0,80)+'"';
try{
var idoc=f.contentDocument;
if(idoc){
var btns=idoc.querySelectorAll('button');
fInfo+=' → ACCESSIBLE ('+btns.length+' buttons)';
// Dump first 5 button texts
for(var b=0;b<Math.min(5,btns.length);b++){
log(' btn['+b+']: "'+((btns[b].textContent||'').trim()).substring(0,40)+'"');
}
}
}catch(e){
fInfo+=' → BLOCKED ('+e.message.substring(0,40)+')';
}
log(fInfo);
}
// Count webview elements
var webviews=document.querySelectorAll('webview');
log('DOM-DUMP: '+webviews.length+' <webview> elements');
for(var w=0;w<webviews.length;w++){
var wInfo=' webview#'+w+' src="'+(webviews[w].src||'').substring(0,80)+'"';
try{
var wdoc=webviews[w].contentDocument;
if(wdoc)wInfo+=' → ACCESSIBLE';
else wInfo+=' → null contentDocument';
}catch(e){wInfo+=' → BLOCKED ('+e.message.substring(0,40)+')';}
log(wInfo);
}
// Main doc buttons
var mainBtns=document.querySelectorAll('button');
log('DOM-DUMP: '+mainBtns.length+' buttons in main document');
for(var m=0;m<Math.min(10,mainBtns.length);m++){
log(' main-btn['+m+']: "'+((mainBtns[m].textContent||'').trim()).substring(0,50)+'"');
}
// Report to bridge
fetch(BASE+'/ping?dom_dump='+iframes.length+'f_'+webviews.length+'wv_'+mainBtns.length+'btn').catch(function(){});
}catch(e){log('DOM-DUMP error: '+e.message);}
}
// ── Port Discovery: async fetch()-based (sync XHR blocked in Electron renderer) ──
var HARDCODED_PORT=${_port};
@@ -587,7 +693,7 @@ function generateApprovalObserverScript(_port: number): string {
discoverPort(function(port){
BASE='http://127.0.0.1:'+port;
fetch(BASE+'/ping').then(function(r){return r.text();}).then(function(t){
if(t==='pong'){log('Bridge connected on port '+port);_ready=true;startObserver();}
if(t==='pong'){log('Bridge connected on port '+port);_ready=true;startObserver();setTimeout(dumpDOMStructure,3000);}
else log('Bridge ping failed: '+t);
}).catch(function(e){log('Bridge unreachable: '+e.message);});
});
@@ -843,49 +949,100 @@ function generateApprovalObserverScript(_port: number): string {
// ── TRIGGER-CLICK: Extension→Renderer bridge for programmatic button clicks ──
// Extension sets clickTrigger via tryApprovalStrategies → renderer polls and clicks
// v3: uses deepFindButtons() to traverse iframes, webviews, shadow DOMs
setInterval(function(){
if(!_ready||!BASE)return;
fetch(BASE+'/trigger-click?t='+Date.now()).then(function(r){return r.json();}).then(function(d){
if(!d.action)return;
log('🔔 TRIGGER-CLICK received: action='+d.action);
// Find first visible approve or reject button
var allBtns=document.querySelectorAll('button');
if(d.action==='approve'){
// Click first Run/Accept/Allow/Continue button
var approveRe=[/^Run/i,/^Accept/i,/^Allow/i,/^Approve/i,/^Continue$/i,/^Proceed$/i];
for(var i=0;i<allBtns.length;i++){
var b=allBtns[i];
if(b.disabled||b.hidden||!b.offsetParent)continue;
var txt=(b.textContent||'').trim();
for(var p=0;p<approveRe.length;p++){
if(approveRe[p].test(txt)){
log('✅ TRIGGER-CLICK: clicking "'+txt+'"');
b.click();
return;
var approveRe=[/^Run$/i,/^Run /i,/^Accept/i,/^Allow/i,/^Approve/i,/^Continue$/i,/^Proceed$/i];
var rejectRe=[/^Reject/i,/^Cancel$/i,/^Deny$/i,/^Stop$/i,/^Decline$/i];
var patterns=(d.action==='approve')?approveRe:rejectRe;
var emoji=(d.action==='approve')?'✅':'❌';
// Phase 1: deepFindButtons in main doc + accessible iframes + shadow DOMs
var found=deepFindButtons(patterns);
if(found.length>0){
log(emoji+' TRIGGER-CLICK: clicking "'+found[0].text+'" from '+found[0].source);
found[0].btn.click();
return;
}
// Phase 2: Try <webview>.executeJavaScript for inaccessible webviews
var webviews=document.querySelectorAll('webview');
if(webviews.length>0){
log('TRIGGER-CLICK: trying '+webviews.length+' webview(s) via executeJavaScript...');
var patternsStr=patterns.map(function(re){return re.source;}).join('|');
var clickScript='(function(){'+
'var re=new RegExp("'+patternsStr+'","i");'+
'var btns=document.querySelectorAll("button");'+
'for(var i=0;i<btns.length;i++){'+
'var b=btns[i];if(b.disabled||b.hidden)continue;'+
'var t=(b.textContent||"").trim();'+
'if(re.test(t)){b.click();return "CLICKED:"+t;}'+
'}'+
'return "NOT_FOUND:"+btns.length+"_buttons";'+
'})()';
for(var w=0;w<webviews.length;w++){
(function(wv,idx){
try{
if(typeof wv.executeJavaScript==='function'){
wv.executeJavaScript(clickScript).then(function(result){
log(emoji+' TRIGGER-CLICK webview#'+idx+': '+result);
}).catch(function(e){
log('TRIGGER-CLICK webview#'+idx+' execJS error: '+e.message);
});
}
}catch(e){
log('TRIGGER-CLICK webview#'+idx+' error: '+e.message);
}
}
})(webviews[w],w);
}
log('⚠️ TRIGGER-CLICK: no approve button found in DOM');
} else if(d.action==='reject'){
for(var j=0;j<allBtns.length;j++){
var b2=allBtns[j];
if(b2.disabled||b2.hidden||!b2.offsetParent)continue;
var txt2=(b2.textContent||'').trim();
for(var r=0;r<REJECT_RE.length;r++){
if(REJECT_RE[r].test(txt2)){
log('❌ TRIGGER-CLICK: clicking "'+txt2+'"');
b2.click();
return;
}
// Phase 3: Try iframes via postMessage (cross-origin fallback)
var iframes=document.querySelectorAll('iframe');
if(iframes.length>0){
log('TRIGGER-CLICK: trying '+iframes.length+' iframe(s) — checking accessibility...');
var clickedAny=false;
for(var fi=0;fi<iframes.length;fi++){
try{
var idoc=iframes[fi].contentDocument||iframes[fi].contentWindow.document;
if(!idoc)continue;
var ibtns=idoc.querySelectorAll('button');
for(var bi=0;bi<ibtns.length;bi++){
var ib=ibtns[bi];
if(ib.disabled||ib.hidden)continue;
var itxt=(ib.textContent||'').trim();
for(var pi=0;pi<patterns.length;pi++){
if(patterns[pi].test(itxt)){
log(emoji+' TRIGGER-CLICK iframe#'+fi+': clicking "'+itxt+'"');
ib.click();
clickedAny=true;
return;
}
}
}
}
}catch(e){}
}
log('⚠️ TRIGGER-CLICK: no reject button found in DOM');
}
if(!found.length){
// Log what we DID find for debugging
var allBtns=document.querySelectorAll('button');
var btnTexts=[];
for(var di=0;di<Math.min(10,allBtns.length);di++){
btnTexts.push('"'+((allBtns[di].textContent||'').trim()).substring(0,30)+'"');
}
log('⚠️ TRIGGER-CLICK: no '+d.action+' button found. Main DOM has '+allBtns.length+' btns: ['+btnTexts.join(',')+']');
log('⚠️ iframes='+document.querySelectorAll('iframe').length+' webviews='+document.querySelectorAll('webview').length);
}
}).catch(function(){});
},1000);
_obs=true;
log('v2 Observer active — MutationObserver + 3s fallback + trigger-click polling');
log('v3 Observer active — deep DOM traversal + MutationObserver + trigger-click polling');
}
})();
`;