Skip to content

Commit 1fe5103

Browse files
Cheshirikskgevorkyan
Cheshiriks
and
kgevorkyan
authored
Test for actual warning in SAVE from the file (#366)
* Support reading of actual warning in SAVE from the file What's done: * Added reading of actual warning from the file Co-authored-by: Kirill Gevorkyan <[email protected]>
1 parent 9074bba commit 1fe5103

File tree

6 files changed

+175
-19
lines changed

6 files changed

+175
-19
lines changed

‎examples/kotlin-diktat/sarif-actual-expected/save-warnings-actual.sarif‎

Lines changed: 50 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,50 @@
1+
{
2+
"$schema": "https://raw.githubusercontent.com/oasis-tcs/sarif-spec/master/Schemata/sarif-schema-2.1.0.json",
3+
"version": "2.1.0",
4+
"runs": [
5+
{
6+
"originalUriBaseIds": {
7+
"%SRCROOT%": {
8+
"uri": "file://D:/projects/"
9+
}
10+
},
11+
"results": [
12+
{
13+
"level": "error",
14+
"locations": [
15+
{
16+
"physicalLocation": {
17+
"artifactLocation": {
18+
"uri": "src\\kotlin\\EnumValueSnakeCaseTest.kt",
19+
"uriBaseId": "%SRCROOT%"
20+
},
21+
"region": {
22+
"startColumn": 5,
23+
"startLine": 18
24+
}
25+
}
26+
}
27+
],
28+
"message": {
29+
"text": "[ENUM_VALUE] enum values should be in selected UPPER_CASE snake/PascalCase format: NAme_MYa_sayR_"
30+
},
31+
"ruleId": "diktat-ruleset:identifier-naming"
32+
}
33+
],
34+
"tool": {
35+
"driver": {
36+
"downloadUri": "https://github.com/pinterest/ktlint/releases/tag/0.42.0",
37+
"fullName": "ktlint",
38+
"informationUri": "https://github.com/pinterest/ktlint/",
39+
"language": "en",
40+
"name": "ktlint",
41+
"organization": "pinterest",
42+
"rules": [
43+
],
44+
"semanticVersion": "0.42.0",
45+
"version": "0.42.0"
46+
}
47+
}
48+
}
49+
]
50+
}

‎examples/kotlin-diktat/sarif-actual-expected/save-warnings-expected.sarif‎

Lines changed: 50 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,50 @@
1+
{
2+
"$schema": "https://raw.githubusercontent.com/oasis-tcs/sarif-spec/master/Schemata/sarif-schema-2.1.0.json",
3+
"version": "2.1.0",
4+
"runs": [
5+
{
6+
"originalUriBaseIds": {
7+
"%SRCROOT%": {
8+
"uri": "file://D:/projects/"
9+
}
10+
},
11+
"results": [
12+
{
13+
"level": "error",
14+
"locations": [
15+
{
16+
"physicalLocation": {
17+
"artifactLocation": {
18+
"uri": "src\\kotlin\\EnumValueSnakeCaseTest.kt",
19+
"uriBaseId": "%SRCROOT%"
20+
},
21+
"region": {
22+
"startColumn": 5,
23+
"startLine": 18
24+
}
25+
}
26+
}
27+
],
28+
"message": {
29+
"text": "[ENUM_VALUE] enum values should be in selected UPPER_CASE snake/PascalCase format: NAme_MYa_sayR_"
30+
},
31+
"ruleId": "diktat-ruleset:identifier-naming"
32+
}
33+
],
34+
"tool": {
35+
"driver": {
36+
"downloadUri": "https://github.com/pinterest/ktlint/releases/tag/0.42.0",
37+
"fullName": "ktlint",
38+
"informationUri": "https://github.com/pinterest/ktlint/",
39+
"language": "en",
40+
"name": "ktlint",
41+
"organization": "pinterest",
42+
"rules": [
43+
],
44+
"semanticVersion": "0.42.0",
45+
"version": "0.42.0"
46+
}
47+
}
48+
}
49+
]
50+
}

‎examples/kotlin-diktat/sarif-actual-expected/save.toml‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,21 @@
1+
[general]
2+
tags = ["warn", "sarif"]
3+
description = "Test warnings discovered by diKTat, reading expected warnings from SARIF file"
4+
suiteName = "Only Warnings: with SARIF format"
5+
6+
[warn]
7+
# regular expression to detect tests
8+
testNameRegex = ".*Test.kt"
9+
expectedWarningsFormat = "SARIF"
10+
expectedWarningsFileName = "save-warnings-expected.sarif"
11+
actualWarningsFormat = "SARIF"
12+
actualWarningsFileName = "save-warnings-actual.sarif"
13+
# diktat warnings have the following format: filename.kt:1:15: [WARN_NAME] my warning
14+
actualWarningsPattern="(\\w+\\..+):(\\d+):(\\d+): (\\[.*\\].*)"
15+
fileNameCaptureGroupOut = 1
16+
lineCaptureGroupOut = 2
17+
columnCaptureGroupOut = 3
18+
messageCaptureGroupOut = 4
19+
exactWarningsMatch = false
20+
warningTextHasColumn = true
21+
warningTextHasLine = true

‎examples/kotlin-diktat/sarif-actual-expected/src/kotlin/EnumValueSnakeCaseTest.kt‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
package org.cqfn.diktat.test.resources.test.paragraph1.naming.enum_
2+
3+
// ;warn:3:1: [MISSING_KDOC_TOP_LEVEL] all public and internal top-level classes and functions should have Kdoc: EnumValueSnakeCaseTest (cannot be auto-corrected)
4+
// ;warn:35: [WRONG_DECLARATIONS_ORDER] declarations of constants and enum members should be sorted alphabetically: enum entries order is incorrect
5+
// ;warn:10:5: [ENUMS_SEPARATED] enum is incorrectly formatted: enums must end with semicolon
6+
enum class EnumValueSnakeCaseTest {
7+
// ;warn:$line+1:5: [ENUM_VALUE] enum values should be{{ in }}selected UPPER_CASE snake/PascalCase format: paSC_SAl_l
8+
paSC_SAl_l,
9+
10+
// ;warn:5: [ENUM_VALUE] enum values{{ should }}be in selected{{ UPPER_CASE }}snake/PascalCase format: PascAsl_f
11+
PascAsl_f
12+
// ;warn:$line-2:5: [ENUMS_SEPARATED] enum is incorrectly formatted: last enum entry must end with a comma
13+
14+
// ;warn:1:9: {{.*}}[PACKAGE_NAME_INCORRECT_PREFIX] package name should start from company's domain: org.cqfn.save{{.*}}
15+
}

‎save-core/src/commonNonJsTest/kotlin/com/saveourtool/save/core/integration/ClassicWarnTest.kt‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -157,4 +157,12 @@ class ClassicWarnTest {
157157
1
158158
)
159159
}
160+
161+
@Test
162+
fun `execute warn-plugin with actual and expected warnings from SARIF`() {
163+
runTestsWithDiktat(
164+
listOf("sarif-actual-expected"),
165+
1
166+
)
167+
}
160168
}

‎save-plugins/warn-plugin/src/commonMain/kotlin/com/saveourtool/save/plugin/warn/WarnPlugin.kt‎

Lines changed: 31 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@ import com.saveourtool.save.core.utils.ExecutionResult
2020
import com.saveourtool.save.core.utils.ProcessExecutionException
2121
import com.saveourtool.save.core.utils.ProcessTimeoutException
2222
import com.saveourtool.save.core.utils.SarifParsingException
23+
import com.saveourtool.save.core.utils.calculatePathToSarifFile
2324
import com.saveourtool.save.core.utils.singleIsInstance
2425
import com.saveourtool.save.plugin.warn.sarif.toWarnings
2526
import com.saveourtool.save.plugin.warn.utils.CmdExecutorWarn
@@ -34,7 +35,6 @@ import com.saveourtool.save.plugin.warn.utils.getLineNumber
3435
import io.github.detekt.sarif4k.SarifSchema210
3536
import okio.FileSystem
3637
import okio.Path
37-
import okio.Path.Companion.toPath
3838

3939
import kotlin.random.Random
4040
import kotlinx.serialization.decodeFromString
@@ -46,6 +46,7 @@ private typealias WarningMap = Map<String, List<Warning>>
4646
* A plugin that runs an executable and verifies that it produces required warning messages.
4747
* @property testConfig
4848
*/
49+
@Suppress("TooManyFunctions")
4950
class WarnPlugin(
5051
testConfig: TestConfig,
5152
testFiles: List<String>,
@@ -151,26 +152,20 @@ class WarnPlugin(
151152
warnMissingExpectedWarnings(warnPluginConfig, generalConfig, originalPaths)
152153
}
153154

154-
val result = try {
155-
cmdExecutor.execCmdAndGetExecutionResults(redirectTo)
156-
} catch (ex: ProcessTimeoutException) {
157-
logWarn("The following tests took too long to run and were stopped: $originalPaths, timeout for single test: ${ex.timeoutMillis}")
158-
return failTestResult(originalPaths, ex, execCmd)
159-
} catch (ex: ProcessExecutionException) {
160-
return failTestResult(originalPaths, ex, execCmd)
161-
}
162-
163-
val actualWarningsMap = try {
164-
warnPluginConfig.actualWarningsFileName?.let {
165-
val execResult = ExecutionResult(
166-
result.code,
167-
fs.readLines(warnPluginConfig.actualWarningsFileName.toPath()),
168-
result.stderr
169-
)
170-
collectActualWarningsWithLineNumbers(execResult, warnPluginConfig, workingDirectory)
171-
} ?: collectActualWarningsWithLineNumbers(result, warnPluginConfig, workingDirectory)
155+
val (actualWarningsMap, result) = try {
156+
actualWarningsIfExistActualWarningsFile(warnPluginConfig, originalPaths, workingDirectory)
172157
} catch (ex: SarifParsingException) {
173158
return failTestResult(originalPaths, ex, execCmd)
159+
} ?: run {
160+
val result = try {
161+
cmdExecutor.execCmdAndGetExecutionResults(redirectTo)
162+
} catch (ex: ProcessTimeoutException) {
163+
logWarn("The following tests took too long to run and were stopped: $originalPaths, timeout for single test: ${ex.timeoutMillis}")
164+
return failTestResult(originalPaths, ex, execCmd)
165+
} catch (ex: ProcessExecutionException) {
166+
return failTestResult(originalPaths, ex, execCmd)
167+
}
168+
collectActualWarningsWithLineNumbers(result, warnPluginConfig, workingDirectory) to result
174169
}
175170

176171
val resultsChecker = ResultsChecker(
@@ -195,6 +190,23 @@ class WarnPlugin(
195190
}.asSequence()
196191
}
197192

193+
private fun actualWarningsIfExistActualWarningsFile(
194+
warnPluginConfig: WarnPluginConfig,
195+
originalPaths: List<Path>,
196+
workingDirectory: Path,
197+
) = warnPluginConfig.actualWarningsFileName?.let {
198+
val sarif = calculatePathToSarifFile(
199+
sarifFileName = warnPluginConfig.actualWarningsFileName,
200+
anchorTestFilePath = originalPaths.first()
201+
)
202+
val execResult = ExecutionResult(
203+
0,
204+
fs.readLines(sarif),
205+
listOf("Warnings were obtained from SARIF file, no debug info is available")
206+
)
207+
collectActualWarningsWithLineNumbers(execResult, warnPluginConfig, workingDirectory) to execResult
208+
}
209+
198210
private fun createTestFiles(paths: List<Path>, warnPluginConfig: WarnPluginConfig): List<Path> {
199211
logDebug("Creating temp copy files of resources for WarnPlugin...")
200212
logTrace("Trying to create temp files for: $paths")

0 commit comments

Comments
 (0)