AI assistance pretty much integrated. 2 options currently that actually work, Claud and Ollama. More planned for later, so other options available they just don't do anything.
This commit is contained in:
Storm Dragon
@@ -56,10 +56,9 @@ class AIAssistant(Plugin):
|
||||
except:
|
||||
pass
|
||||
|
||||
# Keybinding storage
|
||||
self._kb_binding_activate = None
|
||||
self._kb_binding_question = None
|
||||
self._kb_binding_describe = None
|
||||
# Menu and keybinding storage
|
||||
self._kb_binding_menu = None
|
||||
self._menu_gui = None
|
||||
|
||||
# AI provider and settings
|
||||
self._provider_type = None
|
||||
@@ -238,41 +237,144 @@ class AIAssistant(Plugin):
|
||||
return False
|
||||
|
||||
def _register_keybindings(self):
|
||||
"""Register AI Assistant keybindings."""
|
||||
"""Register AI Assistant menu keybinding."""
|
||||
try:
|
||||
# Main AI Assistant activation - avoid conflict with Actions
|
||||
self._kb_binding_activate = self.registerGestureByString(
|
||||
self._handle_ai_activate,
|
||||
"Activate AI Assistant",
|
||||
# Single keybinding to show AI Assistant menu
|
||||
self._kb_binding_menu = self.registerGestureByString(
|
||||
self._show_ai_menu,
|
||||
"Show AI Assistant menu",
|
||||
'kb:cthulhu+control+shift+a'
|
||||
)
|
||||
|
||||
# Ask question about current focus
|
||||
self._kb_binding_question = self.registerGestureByString(
|
||||
self._handle_ai_question,
|
||||
"Ask AI about current focus",
|
||||
'kb:cthulhu+control+shift+q'
|
||||
)
|
||||
|
||||
# Describe current screen
|
||||
self._kb_binding_describe = self.registerGestureByString(
|
||||
self._handle_ai_describe,
|
||||
"AI describe current screen",
|
||||
'kb:cthulhu+control+shift+d'
|
||||
)
|
||||
|
||||
logger.info("AI Assistant keybindings registered")
|
||||
print(f"DEBUG: AI Assistant keybindings registered - activate: {self._kb_binding_activate}, question: {self._kb_binding_question}, describe: {self._kb_binding_describe}")
|
||||
logger.info("AI Assistant menu keybinding registered")
|
||||
print(f"DEBUG: AI Assistant menu keybinding registered: {self._kb_binding_menu}")
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error registering AI keybindings: {e}")
|
||||
logger.error(f"Error registering AI menu keybinding: {e}")
|
||||
|
||||
def _unregister_keybindings(self):
|
||||
"""Unregister AI Assistant keybindings."""
|
||||
# Keybindings are automatically cleaned up when plugin deactivates
|
||||
self._kb_binding_activate = None
|
||||
self._kb_binding_question = None
|
||||
self._kb_binding_describe = None
|
||||
self._kb_binding_menu = None
|
||||
self._menu_gui = None
|
||||
|
||||
def _show_ai_menu(self, script=None, inputEvent=None):
|
||||
"""Show the AI Assistant menu."""
|
||||
try:
|
||||
logger.info("Showing AI Assistant menu")
|
||||
|
||||
# IMPORTANT: Capture screen data BEFORE showing menu
|
||||
# This ensures we get the actual screen content, not the menu itself
|
||||
self._pre_menu_screen_data = self._collect_ai_data()
|
||||
logger.info("Pre-captured screen data for menu actions")
|
||||
|
||||
# Now show the menu
|
||||
self._menu_gui = AIAssistantMenu(self._handle_menu_selection)
|
||||
self._menu_gui.show_gui()
|
||||
return True
|
||||
except Exception as e:
|
||||
logger.error(f"Error showing AI menu: {e}")
|
||||
import traceback
|
||||
traceback.print_exc()
|
||||
return False
|
||||
|
||||
def _handle_menu_selection(self, action_id):
|
||||
"""Handle AI Assistant menu selection."""
|
||||
try:
|
||||
logger.info(f"AI menu selection: {action_id}")
|
||||
|
||||
if action_id == "ask_question":
|
||||
self._handle_ai_question_with_data(self._pre_menu_screen_data)
|
||||
elif action_id == "describe_screen":
|
||||
self._handle_ai_describe_with_data(self._pre_menu_screen_data)
|
||||
elif action_id == "request_action":
|
||||
self._handle_ai_activate_with_data(self._pre_menu_screen_data)
|
||||
else:
|
||||
logger.warning(f"Unknown AI menu action: {action_id}")
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error handling menu selection {action_id}: {e}")
|
||||
|
||||
def _handle_ai_describe_with_data(self, data):
|
||||
"""Handle AI screen description request with pre-captured data."""
|
||||
try:
|
||||
logger.info("AI screen description requested with pre-captured data")
|
||||
|
||||
if not self._enabled:
|
||||
self._present_message("AI Assistant is not enabled")
|
||||
return True
|
||||
|
||||
# Use AI to describe the current screen
|
||||
if not self._ai_provider:
|
||||
self._present_message("AI provider not available. Check configuration.")
|
||||
return True
|
||||
|
||||
self._present_message("AI Assistant analyzing screen...")
|
||||
|
||||
# Use pre-captured data
|
||||
if data:
|
||||
try:
|
||||
response = self._ai_provider.describe_screen(
|
||||
data.get('screenshot'),
|
||||
data.get('accessibility')
|
||||
)
|
||||
self._show_description_dialog(response)
|
||||
except Exception as e:
|
||||
logger.error(f"Error getting AI screen description: {e}")
|
||||
self._present_message(f"Error getting AI screen description: {e}")
|
||||
else:
|
||||
self._present_message("Could not collect screen data for analysis")
|
||||
|
||||
return True
|
||||
except Exception as e:
|
||||
logger.error(f"Error in AI describe handler: {e}")
|
||||
return False
|
||||
|
||||
def _handle_ai_question_with_data(self, data):
|
||||
"""Handle AI question request with pre-captured data."""
|
||||
try:
|
||||
logger.info("AI question requested with pre-captured data")
|
||||
|
||||
if not self._enabled:
|
||||
self._present_message("AI Assistant is not enabled")
|
||||
return True
|
||||
|
||||
if not self._ai_provider:
|
||||
self._present_message("AI provider not available. Check configuration.")
|
||||
return True
|
||||
|
||||
# Store the pre-captured data for use by the question dialog
|
||||
self._current_screen_data = data
|
||||
|
||||
# Show question dialog
|
||||
self._show_question_dialog()
|
||||
return True
|
||||
except Exception as e:
|
||||
logger.error(f"Error in AI question handler: {e}")
|
||||
return False
|
||||
|
||||
def _handle_ai_activate_with_data(self, data):
|
||||
"""Handle AI action request with pre-captured data."""
|
||||
try:
|
||||
logger.info("AI action requested with pre-captured data")
|
||||
|
||||
if not self._enabled:
|
||||
self._present_message("AI Assistant is not enabled")
|
||||
return True
|
||||
|
||||
if not self._ai_provider:
|
||||
self._present_message("AI provider not available. Check configuration.")
|
||||
return True
|
||||
|
||||
# Store the pre-captured data for use by the action dialog
|
||||
self._current_screen_data = data
|
||||
|
||||
# Show action dialog
|
||||
self._show_action_dialog()
|
||||
return True
|
||||
except Exception as e:
|
||||
logger.error(f"Error in AI action handler: {e}")
|
||||
return False
|
||||
|
||||
def _handle_ai_activate(self, script=None, inputEvent=None):
|
||||
"""Handle main AI Assistant activation - now shows action dialog."""
|
||||
@@ -361,7 +463,7 @@ class AIAssistant(Plugin):
|
||||
data.get('screenshot'),
|
||||
data.get('accessibility')
|
||||
)
|
||||
self._present_message(response)
|
||||
self._show_description_dialog(response)
|
||||
except Exception as e:
|
||||
logger.error(f"Error getting AI screen description: {e}")
|
||||
self._present_message(f"Error getting AI screen description: {e}")
|
||||
@@ -779,6 +881,51 @@ class AIAssistant(Plugin):
|
||||
self._present_message(f"Error processing question: {e}")
|
||||
|
||||
# ============================================================================
|
||||
def _show_description_dialog(self, description):
|
||||
"""Show a read-only dialog with the screen description."""
|
||||
try:
|
||||
dialog = Gtk.Dialog(
|
||||
title="AI Screen Description",
|
||||
parent=None,
|
||||
flags=Gtk.DialogFlags.MODAL,
|
||||
buttons=(Gtk.STOCK_CLOSE, Gtk.ResponseType.CLOSE)
|
||||
)
|
||||
|
||||
dialog.set_default_size(600, 400)
|
||||
|
||||
content_area = dialog.get_content_area()
|
||||
|
||||
# Create scrollable text view
|
||||
scrolled_window = Gtk.ScrolledWindow()
|
||||
scrolled_window.set_policy(Gtk.PolicyType.AUTOMATIC, Gtk.PolicyType.AUTOMATIC)
|
||||
scrolled_window.set_shadow_type(Gtk.ShadowType.IN)
|
||||
|
||||
text_view = Gtk.TextView()
|
||||
text_view.set_editable(False) # Read-only
|
||||
text_view.set_cursor_visible(False)
|
||||
text_view.set_wrap_mode(Gtk.WrapMode.WORD)
|
||||
|
||||
# Set the description text
|
||||
text_buffer = text_view.get_buffer()
|
||||
text_buffer.set_text(description)
|
||||
|
||||
scrolled_window.add(text_view)
|
||||
content_area.pack_start(scrolled_window, True, True, 10)
|
||||
|
||||
dialog.set_default_response(Gtk.ResponseType.CLOSE)
|
||||
dialog.show_all()
|
||||
|
||||
# Focus the text view so screen reader reads the content
|
||||
text_view.grab_focus()
|
||||
|
||||
# Show dialog and wait for user to close
|
||||
response = dialog.run()
|
||||
dialog.destroy()
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error showing description dialog: {e}")
|
||||
self._present_message(f"Error showing description: {e}")
|
||||
|
||||
# NEW: Action System Methods for Phase 5
|
||||
# ============================================================================
|
||||
|
||||
@@ -1545,3 +1692,56 @@ class AIAssistant(Plugin):
|
||||
except Exception as e:
|
||||
logger.error(f"Error extracting text from action: {e}")
|
||||
return None
|
||||
|
||||
|
||||
class AIAssistantMenu(Gtk.Menu):
|
||||
"""A menu containing AI Assistant options."""
|
||||
|
||||
def __init__(self, handler):
|
||||
super().__init__()
|
||||
self.connect("popped-up", self._on_popped_up)
|
||||
self.on_option_selected = handler
|
||||
|
||||
# AI Assistant menu options
|
||||
options = [
|
||||
("ask_question", "Ask Question"),
|
||||
("describe_screen", "Describe Screen"),
|
||||
("request_action", "Request Action")
|
||||
]
|
||||
|
||||
for action_id, label in options:
|
||||
menu_item = Gtk.MenuItem(label=label)
|
||||
menu_item.connect("activate", self._on_activate, action_id)
|
||||
self.append(menu_item)
|
||||
|
||||
def _on_activate(self, widget, action_id):
|
||||
"""Handler for menu item activation."""
|
||||
self.on_option_selected(action_id)
|
||||
|
||||
def _on_popped_up(self, *args):
|
||||
"""Handler for menu popup."""
|
||||
logger.info("AI Assistant menu popped up")
|
||||
|
||||
def show_gui(self):
|
||||
"""Shows the AI Assistant menu."""
|
||||
self.show_all()
|
||||
display = Gdk.Display.get_default()
|
||||
seat = display.get_default_seat()
|
||||
device = seat.get_pointer()
|
||||
screen, x, y = device.get_position()
|
||||
|
||||
event = Gdk.Event.new(Gdk.EventType.BUTTON_PRESS)
|
||||
event.set_screen(screen)
|
||||
event.set_device(device)
|
||||
event.time = Gtk.get_current_event_time()
|
||||
event.x = x
|
||||
event.y = y
|
||||
|
||||
rect = Gdk.Rectangle()
|
||||
rect.x = x
|
||||
rect.y = y
|
||||
rect.width = 1
|
||||
rect.height = 1
|
||||
|
||||
window = Gdk.get_default_root_window()
|
||||
self.popup_at_rect(window, rect, Gdk.Gravity.NORTH_WEST, Gdk.Gravity.NORTH_WEST, event)
|
||||
|
||||
Reference in New Issue
Block a user