Skip to content

Commit b91e4e3

Browse files
authored
feat: Flake Indicator ACI (#23049)
1 parent 61b0795 commit b91e4e3

20 files changed

Lines changed: 621 additions & 146 deletions

File tree

packages/app/cypress/e2e/specs_list_actual_git_repo.cy.ts

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -24,7 +24,7 @@ describe('Spec List - Last updated with git info', () => {
2424
cy.get('[data-cy-row="foo.spec.js"] [data-cy="git-info-row"] svg')
2525
.trigger('mouseenter')
2626

27-
cy.get('.v-popper__popper--shown').contains('Created')
27+
cy.get('.v-popper__popper--shown').should('be.visible').and('contain.text', 'Created')
2828
cy.get('[data-cy-row="foo.spec.js"] [data-cy="git-info-row"] svg')
2929
.trigger('mouseleave')
3030

@@ -36,7 +36,7 @@ describe('Spec List - Last updated with git info', () => {
3636
cy.get('[data-cy-row="blank-contents.spec.js"] [data-cy="git-info-row"] svg')
3737
.trigger('mouseenter')
3838

39-
cy.get('.v-popper__popper--shown').contains('Modified')
39+
cy.get('.v-popper__popper--shown').should('be.visible').and('contain.text', 'Modified')
4040
cy.get('[data-cy-row="blank-contents.spec.js"] [data-cy="git-info-row"] svg')
4141
.trigger('mouseleave')
4242

@@ -48,7 +48,7 @@ describe('Spec List - Last updated with git info', () => {
4848
cy.get('[data-cy-row="dom-container.spec.js"] [data-cy="git-info-row"] svg')
4949
.trigger('mouseenter')
5050

51-
cy.get('.v-popper__popper--shown').contains('add all specs')
51+
cy.get('.v-popper__popper--shown').should('be.visible').and('contain.text', 'add all specs')
5252
cy.get('[data-cy-row="dom-container.spec.js"] [data-cy="git-info-row"] svg')
5353
.trigger('mouseleave')
5454

@@ -68,7 +68,7 @@ describe('Spec List - Last updated with git info', () => {
6868
cy.get('[data-cy-row="dom-container.spec.js"] [data-cy="git-info-row"] svg')
6969
.trigger('mouseenter')
7070

71-
cy.get('.v-popper__popper--shown').contains('Modified')
71+
cy.get('.v-popper__popper--shown').should('be.visible').and('contain.text', 'Modified')
7272
cy.get('[data-cy-row="dom-container.spec.js"] [data-cy="git-info-row"] svg')
7373
.trigger('mouseleave')
7474

@@ -89,7 +89,7 @@ describe('Spec List - Last updated with git info', () => {
8989
cy.get('[data-cy-row="foo.spec.js"] [data-cy="git-info-row"] svg')
9090
.trigger('mouseenter')
9191

92-
cy.get('.v-popper__popper--shown').contains('Created')
92+
cy.get('.v-popper__popper--shown').should('be.visible').and('contain.text', 'Created')
9393
cy.get('[data-cy-row="foo.spec.js"] [data-cy="git-info-row"] svg')
9494
.trigger('mouseleave')
9595

@@ -103,7 +103,7 @@ describe('Spec List - Last updated with git info', () => {
103103
cy.get('[data-cy-row="dom-container.spec.js"] [data-cy="git-info-row"] svg')
104104
.trigger('mouseenter')
105105

106-
cy.get('.v-popper__popper--shown').contains('add all specs')
106+
cy.get('.v-popper__popper--shown').should('be.visible').and('contain.text', 'add all specs')
107107
cy.get('[data-cy-row="dom-container.spec.js"] [data-cy="git-info-row"] svg')
108108
.trigger('mouseleave')
109109
})
Lines changed: 104 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,104 @@
1+
describe('App: Spec List - Flaky Indicator', () => {
2+
beforeEach(() => {
3+
cy.scaffoldProject('cypress-in-cypress')
4+
cy.openProject('cypress-in-cypress')
5+
cy.startAppServer('e2e')
6+
cy.loginUser()
7+
8+
cy.withCtx((ctx, o) => {
9+
// Must have a cloud project ID in order to fetch flaky data
10+
o.sinon.stub(ctx.project, 'projectId').resolves('abc123')
11+
// Must have an active Git branch in order to fetch flaky data (see @include($hasBranch) restriction)
12+
o.sinon.stub(ctx.lifecycleManager.git!, 'currentBranch').value('fakeBranch')
13+
})
14+
15+
cy.remoteGraphQLIntercept(async (obj) => {
16+
await new Promise((r) => setTimeout(r, 20))
17+
if (obj.result.data && 'cloudProjectBySlug' in obj.result.data) {
18+
obj.result.data.cloudProjectBySlug = {
19+
__typename: 'CloudProject',
20+
retrievedAt: new Date().toISOString(),
21+
id: `id${obj.variables.slug}`,
22+
projectId: 'abc123',
23+
}
24+
} else if (obj.result.data && 'cloudSpecByPath' in obj.result.data) {
25+
if (obj.variables.specPath.includes('123.spec.js')) {
26+
obj.result.data.cloudSpecByPath = {
27+
__typename: 'CloudProjectSpec',
28+
id: `id${obj.variables.specPath}`,
29+
retrievedAt: new Date().toISOString(),
30+
averageDuration: null,
31+
specRuns: {
32+
__typename: 'CloudSpecRunConnection',
33+
nodes: [],
34+
},
35+
isConsideredFlaky: true,
36+
flakyStatus: {
37+
__typename: 'CloudProjectSpecFlakyStatus',
38+
severity: 'LOW',
39+
flakyRuns: 2,
40+
flakyRunsWindow: 50,
41+
lastFlaky: 2,
42+
dashboardUrl: '#',
43+
},
44+
}
45+
} else {
46+
obj.result.data.cloudSpecByPath = {
47+
__typename: 'CloudProjectSpec',
48+
id: `id${obj.variables.specPath}`,
49+
retrievedAt: new Date().toISOString(),
50+
averageDuration: null,
51+
specRuns: {
52+
__typename: 'CloudSpecRunConnection',
53+
nodes: [],
54+
},
55+
isConsideredFlaky: false,
56+
flakyStatus: null,
57+
}
58+
}
59+
} else if (obj.result.data && 'cloudLatestRunUpdateSpecData' in obj.result.data) {
60+
obj.result.data.cloudLatestRunUpdateSpecData = {
61+
__typename: 'CloudLatestRunUpdateSpecData',
62+
mostRecentUpdate: new Date('2022-06-10').toISOString(),
63+
pollingInterval: 60,
64+
}
65+
}
66+
67+
return obj.result
68+
})
69+
70+
cy.visitApp()
71+
cy.contains('E2E specs')
72+
})
73+
74+
it('shows the "Flaky" badge on specs considered flaky', () => {
75+
let nonFlakyCounter = 0
76+
let flakyCounter = 0
77+
78+
cy.findAllByTestId('spec-item').each((item) => {
79+
const specName = item.text()
80+
const isFlaky = specName.includes('123.spec.js')
81+
82+
cy.wrap(item).find('[data-cy="flaky-badge"]')
83+
.should(isFlaky ? 'be.visible' : 'not.exist')
84+
85+
isFlaky ? flakyCounter++ : nonFlakyCounter++
86+
})
87+
.then(() => {
88+
expect(nonFlakyCounter).to.be.greaterThan(0, 'Test fails to validate flaky badge does not appear on non-flaky tests')
89+
expect(flakyCounter).to.be.greaterThan(0, 'Test fails to validate flaky badge does appear on flaky tests')
90+
})
91+
})
92+
93+
it('shows correct data on tooltip for flaky tests', () => {
94+
cy.contains('[data-cy="spec-item"]', '123.spec.js').find('.v-popper').trigger('mouseenter')
95+
96+
cy.findByTestId('flaky-spec-summary').within(() => {
97+
cy.contains('123.js')
98+
cy.contains('Low')
99+
cy.contains('4% flaky rate')
100+
cy.contains('2 flaky runs / 50 total')
101+
cy.contains('Last flaky 2 runs ago')
102+
})
103+
})
104+
})
Lines changed: 119 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,119 @@
1+
import { gql, useMutation } from '@urql/vue'
2+
import { Ref, computed, watch } from 'vue'
3+
import { useDebounce } from '@vueuse/core'
4+
import { CloudData_RefetchDocument, SpecsListFragment } from '../generated/graphql'
5+
6+
gql`
7+
mutation CloudData_Refetch ($ids: [ID!]!) {
8+
loadRemoteFetchables(ids: $ids){
9+
id
10+
fetchingStatus
11+
}
12+
}
13+
`
14+
15+
type NonNullCloudSpec = Exclude<SpecsListFragment['cloudSpec'], undefined | null>
16+
17+
/**
18+
* Set up watchers & subscriptions to clear/refetch cloud data based on project state relative to latest data in the cloud
19+
* @param isProjectDisconnected Whether project has a valid connection to the cloud
20+
* @param isOffline Whether app is currently offline (network state)
21+
* @param projectId Cloud project ID (slug)
22+
* @param mostRecentUpdate Date/time of the latest project update in the cloud
23+
* @param displayedSpecs Spec entries currently displayed in the app
24+
* @param allSpecs All project spec entries
25+
* @returns Trigger functions
26+
*/
27+
export function useCloudSpecData (
28+
isProjectDisconnected: Ref<boolean>,
29+
isOffline: Ref<boolean>,
30+
projectId: string | null | undefined,
31+
mostRecentUpdate: Ref<string | null>,
32+
displayedSpecs: Ref<(SpecsListFragment | undefined)[]>,
33+
allSpecs: (SpecsListFragment | undefined)[],
34+
) {
35+
const refetchMutation = useMutation(CloudData_RefetchDocument)
36+
37+
const isCloudSpecOlderThan = (item: NonNullCloudSpec, comparisonDttm: string | null) => {
38+
if (item.data?.__typename !== 'CloudProjectSpec' && item.data?.__typename !== 'CloudProjectSpecNotFound') {
39+
return false
40+
}
41+
42+
if (!item.data.retrievedAt || !comparisonDttm) {
43+
return false
44+
}
45+
46+
return new Date(comparisonDttm).getTime() > new Date(item.data.retrievedAt).getTime()
47+
}
48+
49+
const shouldRefetch = (item: NonNullCloudSpec) => {
50+
if (isOffline.value) {
51+
// Offline, no need to refetch
52+
return false
53+
}
54+
55+
if (item.fetchingStatus === 'NOT_FETCHED' || item.fetchingStatus === undefined) {
56+
// Not yet fetched
57+
return true
58+
}
59+
60+
if (isCloudSpecOlderThan(item, mostRecentUpdate.value)) {
61+
// outdated
62+
return true
63+
}
64+
65+
// nothing new, no need to refetch
66+
return false
67+
}
68+
69+
/**
70+
* Refetch any displayed RemoteFetchable entries that are older than the latest cloudProject update
71+
* or that have not yet been fetched
72+
*/
73+
const fetchDisplayedCloudData = async () => {
74+
const cloudSpecIdsToRefetch = displayedSpecs.value
75+
.map((spec) => spec?.cloudSpec)
76+
.filter((cloudSpec): cloudSpec is NonNullCloudSpec => Boolean(cloudSpec && shouldRefetch(cloudSpec)))
77+
.map((cloudSpec) => cloudSpec.id)
78+
?? []
79+
80+
if (!isProjectDisconnected.value && !refetchMutation.fetching.value && cloudSpecIdsToRefetch.length > 0) {
81+
await refetchMutation.executeMutation({ ids: cloudSpecIdsToRefetch })
82+
}
83+
}
84+
85+
/**
86+
* Refetch any cloudSpec entries that are in an Error state
87+
*/
88+
const refetchFailedCloudData = async () => {
89+
const latestRunsIds = allSpecs
90+
.map((s) => s?.cloudSpec)
91+
.filter((cloudSpec): cloudSpec is NonNullCloudSpec => Boolean(cloudSpec?.fetchingStatus === 'ERRORED'))
92+
.map((cloudSpec) => cloudSpec.id) ?? []
93+
94+
await refetchMutation.executeMutation({ ids: [...latestRunsIds] })
95+
}
96+
97+
const displayedSpecIds = computed(() => displayedSpecs.value.map((v) => v?.cloudSpec?.id).filter((id) => !!id).join('|'))
98+
const debouncedDisplayedSpecIds = useDebounce(displayedSpecIds, 200)
99+
100+
/*
101+
Automatically trigger refresh & purge on the following:
102+
- Set of displayed specs changes (scroll thru virtualized list)
103+
- Network connectivity state changes
104+
- Project connectivity state changes
105+
- Latest update timestamp for cloud project changes
106+
*/
107+
watch(
108+
[debouncedDisplayedSpecIds, isOffline, isProjectDisconnected, mostRecentUpdate],
109+
() => {
110+
fetchDisplayedCloudData()
111+
},
112+
{ flush: 'post' },
113+
)
114+
115+
return {
116+
fetchDisplayedCloudData,
117+
refetchFailedCloudData,
118+
}
119+
}

packages/app/src/specs/SpecItem.vue

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@
2323
class="font-light group-hocus:text-gray-400"
2424
highlight-classes="text-gray-1000"
2525
/>
26+
<slot />
2627
</div>
2728
</div>
2829
</template>

0 commit comments

Comments
 (0)