Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Voice markup #12

Open
wants to merge 7 commits into
base: master
Choose a base branch
from
Open

Voice markup #12

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
53207c6
Voice markup
poretsky Mar 18, 2020
963ba33
Voice markup enhancement
poretsky Mar 21, 2020
1f2e738
Special case of decorated checkable elements
poretsky Mar 24, 2020
afcfa5b
Better fit for narrow screens
poretsky Mar 25, 2020
d6357dc
Unchanged voice for disabled elements
poretsky Mar 25, 2020
ca02f1c
Clickable images are voicified as buttons
poretsky Mar 25, 2020
6aa17bb
Voice for headers
poretsky Mar 26, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
99 changes: 99 additions & 0 deletions compositor/src/main/java/Compositor.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -206,6 +206,8 @@ public class Compositor {
static final int ENUM_RANGE_INFO_TYPE = 6;
static final int RANGE_INFO_UNDEFINED = -1;

private static final int ENUM_VOICE_TYPE = 7;

// Enum values
private static final int QUEUE_MODE_INTERRUPTIBLE_IF_LONG = 0x40000001;

Expand All @@ -225,6 +227,23 @@ public class Compositor {
public static final int DESC_ORDER_STATE_NAME_ROLE_POSITION = 1;
public static final int DESC_ORDER_NAME_ROLE_STATE_POSITION = 2;

/** Voice type ids. */
@IntDef({
VOICE_TYPE_LOW,
VOICE_TYPE_REDUCED,
VOICE_TYPE_NORMAL,
VOICE_TYPE_ELEVATED,
VOICE_TYPE_HIGH
})
@Retention(RetentionPolicy.SOURCE)
public @interface VoiceType {}

public static final int VOICE_TYPE_LOW = 0;
public static final int VOICE_TYPE_REDUCED = 1;
public static final int VOICE_TYPE_NORMAL = 2;
public static final int VOICE_TYPE_ELEVATED = 3;
public static final int VOICE_TYPE_HIGH = 4;

/////////////////////////////////////////////////////////////////////////////////
// Member variables

Expand Down Expand Up @@ -257,6 +276,12 @@ static class Constants {
boolean mSpeakRoles = true;
boolean mSpeakCollectionInfo = true;
@DescriptionOrder int mDescriptionOrder = DESC_ORDER_ROLE_NAME_STATE_POSITION;
@VoiceType int mActionableElementVoice = VOICE_TYPE_NORMAL;
@VoiceType int mButtonVoice = VOICE_TYPE_NORMAL;
@VoiceType int mSwitchVoice = VOICE_TYPE_NORMAL;
@VoiceType int mSelectedItemVoice = VOICE_TYPE_NORMAL;
@VoiceType int mHeaderVoice = VOICE_TYPE_NORMAL;
@VoiceType int mImageVoice = VOICE_TYPE_NORMAL;
boolean mSpeakElementIds = false;
}

Expand Down Expand Up @@ -325,6 +350,48 @@ public void setDescriptionOrder(@DescriptionOrder int descOrderInt) {
}
}

public void setActionableElementVoice(@VoiceType int voiceType) {
if (voiceType != mConstants.mActionableElementVoice) {
mConstants.mActionableElementVoice = voiceType;
mParseTreeIsStale = true;
}
}

public void setButtonVoice(@VoiceType int voiceType) {
if (voiceType != mConstants.mButtonVoice) {
mConstants.mButtonVoice = voiceType;
mParseTreeIsStale = true;
}
}

public void setSwitchVoice(@VoiceType int voiceType) {
if (voiceType != mConstants.mSwitchVoice) {
mConstants.mSwitchVoice = voiceType;
mParseTreeIsStale = true;
}
}

public void setSelectedItemVoice(@VoiceType int voiceType) {
if (voiceType != mConstants.mSelectedItemVoice) {
mConstants.mSelectedItemVoice = voiceType;
mParseTreeIsStale = true;
}
}

public void setHeaderVoice(@VoiceType int voiceType) {
if (voiceType != mConstants.mHeaderVoice) {
mConstants.mHeaderVoice = voiceType;
mParseTreeIsStale = true;
}
}

public void setImageVoice(@VoiceType int voiceType) {
if (voiceType != mConstants.mImageVoice) {
mConstants.mImageVoice = voiceType;
mParseTreeIsStale = true;
}
}

public void setSpeakElementIds(boolean speakElementIds) {
if (speakElementIds != mConstants.mSpeakElementIds) {
mConstants.mSpeakElementIds = speakElementIds;
Expand Down Expand Up @@ -683,6 +750,30 @@ private static void declareConstants(ParseTree parseTree, Constants constants) {
"VERBOSITY_DESCRIPTION_ORDER",
ENUM_VERBOSITY_DESCRIPTION_ORDER,
constants.mDescriptionOrder);
parseTree.setConstantEnum(
"ACTIONABLE_ELEMENT_VOICE",
ENUM_VOICE_TYPE,
constants.mActionableElementVoice);
parseTree.setConstantEnum(
"BUTTON_VOICE",
ENUM_VOICE_TYPE,
constants.mButtonVoice);
parseTree.setConstantEnum(
"SWITCH_VOICE",
ENUM_VOICE_TYPE,
constants.mSwitchVoice);
parseTree.setConstantEnum(
"SELECTED_ITEM_VOICE",
ENUM_VOICE_TYPE,
constants.mSelectedItemVoice);
parseTree.setConstantEnum(
"HEADER_VOICE",
ENUM_VOICE_TYPE,
constants.mHeaderVoice);
parseTree.setConstantEnum(
"IMAGE_VOICE",
ENUM_VOICE_TYPE,
constants.mImageVoice);
parseTree.setConstantBool("VERBOSITY_SPEAK_ELEMENT_IDS", constants.mSpeakElementIds);
}

Expand Down Expand Up @@ -773,6 +864,14 @@ private static void declareEnums(ParseTree parseTree) {
verbosityDescOrderValues.put(DESC_ORDER_NAME_ROLE_STATE_POSITION, "NameRoleStatePosition");
parseTree.addEnum(ENUM_VERBOSITY_DESCRIPTION_ORDER, verbosityDescOrderValues);

Map<Integer, String> voiceTypeValues = new HashMap<>();
voiceTypeValues.put(VOICE_TYPE_LOW, "low");
voiceTypeValues.put(VOICE_TYPE_REDUCED, "reduced");
voiceTypeValues.put(VOICE_TYPE_NORMAL, "normal");
voiceTypeValues.put(VOICE_TYPE_ELEVATED, "elevated");
voiceTypeValues.put(VOICE_TYPE_HIGH, "high");
parseTree.addEnum(ENUM_VOICE_TYPE, voiceTypeValues);

Map<Integer, String> rangeInfoTypes = new HashMap<>();
rangeInfoTypes.put(RangeInfo.RANGE_TYPE_INT, "int");
rangeInfoTypes.put(RangeInfo.RANGE_TYPE_FLOAT, "float");
Expand Down
28 changes: 28 additions & 0 deletions compositor/src/main/java/NodeVariables.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@
import android.view.inputmethod.InputMethodManager;
import android.view.inputmethod.InputMethodSubtype;
import com.google.android.accessibility.utils.AccessibilityNodeInfoUtils;
import com.google.android.accessibility.utils.Filter;
import com.google.android.accessibility.utils.LocaleUtils;
import com.google.android.accessibility.utils.PackageManagerUtils;
import com.google.android.accessibility.utils.Role;
Expand Down Expand Up @@ -115,6 +116,9 @@ class NodeVariables implements ParseTree.VariableDelegate {
private static final int NODE_SELF_MENU_ACTION_TYPE = 7050;
private static final int NODE_IS_WEB_CONTAINER = 7051;

private static final int NODE_CONTAINS_CHECKABLE = 7052;
private static final int NODE_IS_CHECKED_INTERNALLY = 7053;

private final Context mContext;
private final @Nullable LabelManager mLabelManager;
private final ParseTree.VariableDelegate mParentVariables;
Expand Down Expand Up @@ -324,6 +328,10 @@ public boolean getBoolean(int variableId) {
return nodeMenuProvider != null
&& !nodeMenuProvider.getSelfNodeMenuActionTypes(mNode).isEmpty();
}
case NODE_CONTAINS_CHECKABLE:
return getInternalCheckable() != null;
case NODE_IS_CHECKED_INTERNALLY:
return isCheckedInternally();
default:
return mParentVariables.getBoolean(variableId);
}
Expand Down Expand Up @@ -681,6 +689,24 @@ private void createVisitedNodes() {
}
}

private @Nullable AccessibilityNodeInfoCompat getInternalCheckable() {
List<AccessibilityNodeInfoCompat> checkables =
AccessibilityNodeInfoUtils.getMatchingDescendantsOrRoot(mNode, new Filter<AccessibilityNodeInfoCompat>() {
@Override
public boolean accept(AccessibilityNodeInfoCompat node) {
return node != null && node.isCheckable();
}
});
if (checkables != null && checkables.size() == 1)
return checkables.get(0);
return null;
}

private boolean isCheckedInternally() {
AccessibilityNodeInfoCompat node = getInternalCheckable();
return node != null && node.isChecked();
}

static void declareVariables(ParseTree parseTree) {
// Variables.
// Nodes.
Expand Down Expand Up @@ -741,5 +767,7 @@ static void declareVariables(ParseTree parseTree) {
"node.selfMenuActionAvailable", NODE_SELF_MENU_ACTION_IS_AVAILABLE);
parseTree.addStringVariable("node.selfMenuActions", NODE_SELF_MENU_ACTION_TYPE);
parseTree.addBooleanVariable("node.isWebContainer", NODE_IS_WEB_CONTAINER);
parseTree.addBooleanVariable("node.containsCheckable", NODE_CONTAINS_CHECKABLE);
parseTree.addBooleanVariable("node.isCheckedInternally", NODE_IS_CHECKED_INTERNALLY);
}
}
108 changes: 107 additions & 1 deletion compositor/src/main/res/raw/compositor.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -209,6 +209,7 @@
"then":"queue",
"else":"flush"
},
"ttsPitch": "%get_voice_for_element",
"ttsAddToHistory": true,
// TODO: Remove the next line when focus management feature is settled down.
//"ttsForceFeedback": "!$global.syncedAccessibilityFocusLatch && $node.role != 'web_view'",
Expand Down Expand Up @@ -242,6 +243,7 @@
},
"TYPE_VIEW_FOCUSED": {
"ttsOutput": "%event_description",
"ttsPitch": "%get_voice_for_element",
"ttsAddToHistory": true,
// From FallbackFormatter
// TODO: Delete porting comments.
Expand All @@ -256,6 +258,7 @@
},
"TYPE_VIEW_HOVER_ENTER": {
"ttsOutput": "%event_description",
"ttsPitch": "%get_voice_for_element",
"ttsAddToHistory": true,
// TODO: respect user settings
"ttsForceFeedbackAudioPlaybackActive": true,
Expand Down Expand Up @@ -1799,6 +1802,109 @@
}
}
]
}
},
"get_voice_for_element": {
"if": "$node.isEnabled",
"then": "%element_type_specific_voice",
"else": "%voice_normal"
},
"element_type_specific_voice": {
"if": "$node.isSelected || $node.isChecked || $node.isCheckedInternally",
"then": "%selected_item_voice",
"else": {
"if": "$node.isWebContainer && $node.isActionable",
"then": "%actionable_element_voice",
"else": {
"switch": "$node.role",
"cases": {
"button": "%button_voice",
"image_button": "%button_voice",
"radio_button": "%switch_voice",
"toggle_button": "%switch_voice",
"switch": "%switch_voice",
"check_box": "%switch_voice",
"image": {
"if": "$node.isClickable",
"then": "%button_voice",
"else": "%image_voice"
}
},
"default": {
"if": "$node.containsCheckable",
"then": "%switch_voice",
"else": {
"if": "$node.isHeading",
"then": "%header_voice",
"else": "%voice_normal"
}
}
}
}
},
"header_voice": {
"switch": "#HEADER_VOICE",
"cases": {
"low": "%voice_low",
"reduced": "%voice_reduced",
"normal": "%voice_normal",
"elevated": "%voice_elevated",
"high": "%voice_high"
}
},
"actionable_element_voice": {
"switch": "#ACTIONABLE_ELEMENT_VOICE",
"cases": {
"low": "%voice_low",
"reduced": "%voice_reduced",
"normal": "%voice_normal",
"elevated": "%voice_elevated",
"high": "%voice_high"
}
},
"button_voice": {
"switch": "#BUTTON_VOICE",
"cases": {
"low": "%voice_low",
"reduced": "%voice_reduced",
"normal": "%voice_normal",
"elevated": "%voice_elevated",
"high": "%voice_high"
}
},
"switch_voice": {
"switch": "#SWITCH_VOICE",
"cases": {
"low": "%voice_low",
"reduced": "%voice_reduced",
"normal": "%voice_normal",
"elevated": "%voice_elevated",
"high": "%voice_high"
}
},
"selected_item_voice": {
"switch": "#SELECTED_ITEM_VOICE",
"cases": {
"low": "%voice_low",
"reduced": "%voice_reduced",
"normal": "%voice_normal",
"elevated": "%voice_elevated",
"high": "%voice_high"
}
},
"image_voice": {
"switch": "#IMAGE_VOICE",
"cases": {
"low": "%voice_low",
"reduced": "%voice_reduced",
"normal": "%voice_normal",
"elevated": "%voice_elevated",
"high": "%voice_high"
}
},
"voice_low": 0.6,
"voice_reduced": 0.8,
"voice_normal": 1.0,
"voice_elevated": 1.25,
"voice_high": 1.5
}
}
39 changes: 39 additions & 0 deletions talkback/src/main/java/TalkBackService.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1878,6 +1878,27 @@ private void reloadPreferences() {
prefs, res, R.string.pref_node_desc_order_key, R.string.pref_node_desc_order_default);
compositor.setDescriptionOrder(prefValueToDescriptionOrder(res, descriptionOrder));

// Update voice markup preferences.
String voiceType =
SharedPreferencesUtils.getStringPref(
prefs, res, R.string.pref_button_voice_key, R.string.pref_voice_default);
compositor.setButtonVoice(prefValueToVoiceType(res, voiceType));
voiceType = SharedPreferencesUtils.getStringPref(
prefs, res, R.string.pref_switch_voice_key, R.string.pref_voice_default);
compositor.setSwitchVoice(prefValueToVoiceType(res, voiceType));
voiceType = SharedPreferencesUtils.getStringPref(
prefs, res, R.string.pref_selected_item_voice_key, R.string.pref_voice_default);
compositor.setSelectedItemVoice(prefValueToVoiceType(res, voiceType));
voiceType = SharedPreferencesUtils.getStringPref(
prefs, res, R.string.pref_header_voice_key, R.string.pref_voice_default);
compositor.setHeaderVoice(prefValueToVoiceType(res, voiceType));
voiceType = SharedPreferencesUtils.getStringPref(
prefs, res, R.string.pref_image_voice_key, R.string.pref_voice_default);
compositor.setImageVoice(prefValueToVoiceType(res, voiceType));
voiceType = SharedPreferencesUtils.getStringPref(
prefs, res, R.string.pref_actionable_voice_key, R.string.pref_voice_default);
compositor.setActionableElementVoice(prefValueToVoiceType(res, voiceType));

// Update preference: speak element IDs.
boolean speakElementIds =
SharedPreferencesUtils.getBooleanPref(
Expand Down Expand Up @@ -1918,6 +1939,24 @@ private void reloadPreferences() {
}
}

private static @Compositor.VoiceType int prefValueToVoiceType(
Resources resources, String value) {
if (TextUtils.equals(
value, resources.getString(R.string.voice_type_low_pitch))) {
return Compositor.VOICE_TYPE_LOW;
} else if (TextUtils.equals(
value, resources.getString(R.string.voice_type_reduced_pitch))) {
return Compositor.VOICE_TYPE_REDUCED;
} else if (TextUtils.equals(
value, resources.getString(R.string.voice_type_elevated_pitch))) {
return Compositor.VOICE_TYPE_ELEVATED;
} else if (TextUtils.equals(
value, resources.getString(R.string.voice_type_high_pitch))) {
return Compositor.VOICE_TYPE_HIGH;
}
return Compositor.VOICE_TYPE_NORMAL;
}

/**
* Attempts to return the state of touch exploration.
*
Expand Down
Loading