Attachment #820369 for bug #829602

View | Details | Raw Unified | Return to bug 829602 | Differences between
Collapse All | Expand All




++ /clang-tools-extra/clangd/CMakeLists.txt


include(${CMAKE_CURRENT_SOURCE_DIR}/quality/CompletionModel.cmake)
gen_decision_forest(${CMAKE_CURRENT_SOURCE_DIR}/quality/model CompletionModel clang::clangd::Example)
file(GLOB CompletionModel_files "${CMAKE_CURRENT_BINARY_DIR}/CompletionModel/CompletionModel*.cpp")

if(MSVC AND NOT CLANG_CL)
 set_source_files_properties(CompileCommands.cpp PROPERTIES COMPILE_FLAGS -wd4130) # disables C4130: logical operation on address of string constant

  TUScheduler.cpp
  URI.cpp
  XRefs.cpp
  ${CompletionModel_files}

  index/Background.cpp
  index/BackgroundIndexLoader.cpp

  omp_gen
  )

# Include generated Completion Model header.
target_include_directories(clangDaemon PUBLIC
  $<BUILD_INTERFACE:${CMAKE_CURRENT_BINARY_DIR}/CompletionModel>
)

clang_target_link_libraries(clangDaemon
++ /clang-tools-extra/clangd/quality/CompletionModel.cmake

# Run the Completion Model Codegenerator on the model present in the
# ${model} directory.
# Produces files called ${filename}.h and ${filename}*.cpp in the
# ${CMAKE_CURRENT_BINARY_DIR}/${filename} directory. The generated header
# will define a C++ class called ${cpp_class} - which may be a
# namespace-qualified class name.
set(CLANGD_COMPLETION_MODEL_COMPILER ${CMAKE_CURRENT_LIST_DIR}/CompletionModelCodegen.py)
function(gen_decision_forest model filename cpp_class)
  set(model_compiler ${CLANGD_COMPLETION_MODEL_COMPILER})

  set(output_dir ${CMAKE_CURRENT_BINARY_DIR}/${filename})
  set(header_file ${output_dir}/${filename}.h)
  set(cpp_file ${output_dir}/${filename}.cpp)

  # Command for initial (CMake-invocation-time) generation of files.
  message(STATUS "Generating code completion model runtime")
  execute_process(COMMAND "${Python3_EXECUTABLE}" ${model_compiler}
      --model ${model}
      --output_dir ${output_dir}
      --filename ${filename}
      --cpp_class ${cpp_class})

  file(GLOB cpp_files "${output_dir}/${filename}*.cpp")

  # Command for potential regeneration of files after changes in model compiler,
  # forest.json or features.json.
  # This works reliably only if list of generated files is unchanged.
  add_custom_command(OUTPUT ${header_file} ${cpp_files}
    COMMAND "${Python3_EXECUTABLE}" ${model_compiler}
      --model ${model}
      --output_dir ${output_dir}

      --cpp_class ${cpp_class}
    COMMENT "Generating code completion model runtime..."
    DEPENDS ${model_compiler} ${model}/forest.json ${model}/features.json
    VERBATIM)

  set_source_files_properties(${header_file} PROPERTIES
    GENERATED 1)
  set_source_files_properties(${cpp_files} PROPERTIES
    GENERATED 1)

  # Disable unused label warning for generated files.
  if (CMAKE_CXX_COMPILER_ID STREQUAL "MSVC")
    set_source_files_properties(${cpp_files} PROPERTIES
      COMPILE_FLAGS /wd4102)
  else()
    set_source_files_properties(${cpp_files} PROPERTIES
      COMPILE_FLAGS -Wno-unused)
  endif()
endfunction()
++ /clang-tools-extra/clangd/quality/CompletionModelCodegen.py

"""Code generator for Code Completion Model Inference.

Tool runs on the Decision Forest model defined in {model} directory.
It generates files: {output_dir}/{filename}.h, {output_dir}/{filename}.cpp,
and {output_dir}/{filename}{number}.cpp for each Decision Tree.
The generated files define the Example class named {cpp_class} having all the features as class members.
The generated runtime provides an `Evaluate` function which can be used to score a code completion candidate.
"""

import argparse
import json
import os
import struct



    `float Evaluate(const {Example}&)` function. This function can be 
    used to score an Example."""

    functions_codes = {}

    # Generate evaluation function of each tree.
    code += "namespace {\n"
    tree_num = 0
    for tree_json in forest_json:
        functions_codes[f"{tree_num}"] = "LLVM_ATTRIBUTE_NOINLINE float EvaluateTree%d(const %s& E) {\n" % (tree_num, cpp_class.name)
        functions_codes[f"{tree_num}"] += "  " + \
            "\n  ".join(
                tree(tree_json, tree_num=tree_num, node_num=0)[0]) + "\n"
        functions_codes[f"{tree_num}"] += "}\n"
        tree_num += 1
    code += "} // namespace\n\n"

    # Combine the scores of all trees in the final function.
    # MSAN will timeout if these functions are inlined.
    final_function_code = ""
    code += "  float Score = 0;\n"
    for tree_num in range(len(forest_json)):
        final_function_code += "float EvaluateTree%d(const %s& E);\n" % (tree_num, cpp_class.name)
    final_function_code += "\n"
    final_function_code += "float Evaluate(const %s& E) {\n" % cpp_class.name
    final_function_code += "  float Score = 0;\n"
    for tree_num in range(len(forest_json)):
        final_function_code += "  Score += EvaluateTree%d(E);\n" % tree_num
    final_function_code += "  return Score;\n"
    final_function_code += "}\n"

    return functions_codes, final_function_code


def gen_cpp_code(forest_json, features_json, filename, cpp_class):
    """Generates code for the .cpp files."""
    # Headers
    # Required by OrderEncode(float F).
    angled_include = [

        for feature in features_json
        if feature["kind"] == "ENUM")
    nl = "\n"
    return """%s

    functions_codes, final_function_code = evaluate_func(forest_json, cpp_class)

    cpp_code = {
        "": """%s

%s


%s
%s
""" % (nl.join(angled_include), nl.join(quoted_include), cpp_class.ns_begin(),
       cpp_class.name, final_function_code, cpp_class.ns_end())}

    for function_code_num, function_code in functions_codes.items():
        cpp_code[function_code_num] = """%s

#define BIT(X) (1LL << X)

%s

%s

%s
%s
""" % (nl.join(quoted_include), cpp_class.ns_begin(), using_decls,
       function_code, cpp_class.ns_end())

    return cpp_code


def main():

    output_dir = ns.output_dir
    filename = ns.filename
    header_file = "%s/%s.h" % (output_dir, filename)
    cpp_file = "%s/%s.cpp" % (output_dir, filename)
    cpp_class = CppClass(cpp_class=ns.cpp_class)

    model_file = "%s/forest.json" % ns.model

    with open(model_file) as m:
        forest_json = json.load(m)

    os.makedirs(output_dir, exist_ok=True)

    for cpp_code_num, cpp_code in gen_cpp_code(forest_json=forest_json,
        features_json=features_json,
        filename=filename,
        cpp_class=cpp_class).items():
        cpp_file = "%s/%s%s.cpp" % (output_dir, filename, cpp_code_num)
        with open(cpp_file, 'w+t') as output_cc:
            output_cc.write(cpp_code)

    with open(header_file, 'w+t') as output_h:
        output_h.write(gen_header_code(
++ /clang-tools-extra/clangd/unittests/CMakeLists.txt


include(${CMAKE_CURRENT_SOURCE_DIR}/../quality/CompletionModel.cmake)
gen_decision_forest(${CMAKE_CURRENT_SOURCE_DIR}/decision_forest_model DecisionForestRuntimeTest ::ns1::ns2::test::Example)
file(GLOB DecisionForestRuntimeTest_files "${CMAKE_CURRENT_BINARY_DIR}/DecisionForestRuntimeTest/DecisionForestRuntimeTest*.cpp")

add_custom_target(ClangdUnitTests)
add_unittest(ClangdUnitTests ClangdTests

  TypeHierarchyTests.cpp
  URITests.cpp
  XRefsTests.cpp
  ${DecisionForestRuntimeTest_files}

  support/CancellationTests.cpp
  support/ContextTests.cpp

  $<TARGET_OBJECTS:obj.clangDaemonTweaks>
  )

# Include generated Completion Model header.
target_include_directories(ClangdTests PUBLIC
  $<BUILD_INTERFACE:${CMAKE_CURRENT_BINARY_DIR}/DecisionForestRuntimeTest>
)

clang_target_link_libraries(ClangdTests

Return to bug 829602

Line Link Here

(-)file_not_specified_in_diff (-47 / +82 lines)
0	-- /clang-tools-extra/clangd/CMakeLists.txt	0	++ /clang-tools-extra/clangd/CMakeLists.txt
Lines 45-50 Link Here
45		45
46	include(${CMAKE_CURRENT_SOURCE_DIR}/quality/CompletionModel.cmake)	46	include(${CMAKE_CURRENT_SOURCE_DIR}/quality/CompletionModel.cmake)
47	gen_decision_forest(${CMAKE_CURRENT_SOURCE_DIR}/quality/model CompletionModel clang::clangd::Example)	47	gen_decision_forest(${CMAKE_CURRENT_SOURCE_DIR}/quality/model CompletionModel clang::clangd::Example)
		48	file(GLOB CompletionModel_files "${CMAKE_CURRENT_BINARY_DIR}/CompletionModel/CompletionModel*.cpp")
48		49
49	if(MSVC AND NOT CLANG_CL)	50	if(MSVC AND NOT CLANG_CL)
50	set_source_files_properties(CompileCommands.cpp PROPERTIES COMPILE_FLAGS -wd4130) # disables C4130: logical operation on address of string constant	51	set_source_files_properties(CompileCommands.cpp PROPERTIES COMPILE_FLAGS -wd4130) # disables C4130: logical operation on address of string constant
Lines 102-108 Link Here
102	TUScheduler.cpp	103	TUScheduler.cpp
103	URI.cpp	104	URI.cpp
104	XRefs.cpp	105	XRefs.cpp
105	${CMAKE_CURRENT_BINARY_DIR}/CompletionModel.cpp	106	${CompletionModel_files}
106		107
107	index/Background.cpp	108	index/Background.cpp
108	index/BackgroundIndexLoader.cpp	109	index/BackgroundIndexLoader.cpp
Lines 140-148 Link Here
140	omp_gen	141	omp_gen
141	)	142	)
142		143
143	# Include generated CompletionModel headers.	144	# Include generated Completion Model header.
144	target_include_directories(clangDaemon PUBLIC	145	target_include_directories(clangDaemon PUBLIC
145	$<BUILD_INTERFACE:${CMAKE_CURRENT_BINARY_DIR}>	146	$<BUILD_INTERFACE:${CMAKE_CURRENT_BINARY_DIR}/CompletionModel>
146	)	147	)
147		148
148	clang_target_link_libraries(clangDaemon	149	clang_target_link_libraries(clangDaemon
149	-- /clang-tools-extra/clangd/quality/CompletionModel.cmake	150	++ /clang-tools-extra/clangd/quality/CompletionModel.cmake
Lines 1-18 Link Here
1	# Run the Completion Model Codegenerator on the model present in the	1	# Run the Completion Model Codegenerator on the model present in the
2	# ${model} directory.	2	# ${model} directory.
3	# Produces a pair of files called ${filename}.h and ${filename}.cpp in the	3	# Produces files called ${filename}.h and ${filename}*.cpp in the
4	# ${CMAKE_CURRENT_BINARY_DIR}. The generated header	4	# ${CMAKE_CURRENT_BINARY_DIR}/${filename} directory. The generated header
5	# will define a C++ class called ${cpp_class} - which may be a	5	# will define a C++ class called ${cpp_class} - which may be a
6	# namespace-qualified class name.	6	# namespace-qualified class name.
7	set(CLANGD_COMPLETION_MODEL_COMPILER ${CMAKE_CURRENT_LIST_DIR}/CompletionModelCodegen.py)	7	set(CLANGD_COMPLETION_MODEL_COMPILER ${CMAKE_CURRENT_LIST_DIR}/CompletionModelCodegen.py)
8	function(gen_decision_forest model filename cpp_class)	8	function(gen_decision_forest model filename cpp_class)
9	set(model_compiler ${CLANGD_COMPLETION_MODEL_COMPILER})	9	set(model_compiler ${CLANGD_COMPLETION_MODEL_COMPILER})
10		10
11	set(output_dir ${CMAKE_CURRENT_BINARY_DIR})	11	set(output_dir ${CMAKE_CURRENT_BINARY_DIR}/${filename})
12	set(header_file ${output_dir}/${filename}.h)	12	set(header_file ${output_dir}/${filename}.h)
13	set(cpp_file ${output_dir}/${filename}.cpp)
14		13
15	add_custom_command(OUTPUT ${header_file} ${cpp_file}	14	# Command for initial (CMake-invocation-time) generation of files.
		15	message(STATUS "Generating code completion model runtime")
		16	execute_process(COMMAND "${Python3_EXECUTABLE}" ${model_compiler}
		17	--model ${model}
		18	--output_dir ${output_dir}
		19	--filename ${filename}
		20	--cpp_class ${cpp_class})
		21
		22	file(GLOB cpp_files "${output_dir}/${filename}*.cpp")
		23
		24	# Command for potential regeneration of files after changes in model compiler,
		25	# forest.json or features.json.
		26	# This works reliably only if list of generated files is unchanged.
		27	add_custom_command(OUTPUT ${header_file} ${cpp_files}
16	COMMAND "${Python3_EXECUTABLE}" ${model_compiler}	28	COMMAND "${Python3_EXECUTABLE}" ${model_compiler}
17	--model ${model}	29	--model ${model}
18	--output_dir ${output_dir}	30	--output_dir ${output_dir}
Lines 20-38 Link Here
20	--cpp_class ${cpp_class}	32	--cpp_class ${cpp_class}
21	COMMENT "Generating code completion model runtime..."	33	COMMENT "Generating code completion model runtime..."
22	DEPENDS ${model_compiler} ${model}/forest.json ${model}/features.json	34	DEPENDS ${model_compiler} ${model}/forest.json ${model}/features.json
23	VERBATIM )	35	VERBATIM)
24		36
25	set_source_files_properties(${header_file} PROPERTIES	37	set_source_files_properties(${header_file} PROPERTIES
26	GENERATED 1)	38	GENERATED 1)
27	set_source_files_properties(${cpp_file} PROPERTIES	39	set_source_files_properties(${cpp_files} PROPERTIES
28	GENERATED 1)	40	GENERATED 1)
29		41
30	# Disable unused label warning for generated files.	42	# Disable unused label warning for generated files.
31	if (CMAKE_CXX_COMPILER_ID STREQUAL "MSVC")	43	if (CMAKE_CXX_COMPILER_ID STREQUAL "MSVC")
32	set_source_files_properties(${cpp_file} PROPERTIES	44	set_source_files_properties(${cpp_files} PROPERTIES
33	COMPILE_FLAGS /wd4102)	45	COMPILE_FLAGS /wd4102)
34	else()	46	else()
35	set_source_files_properties(${cpp_file} PROPERTIES	47	set_source_files_properties(${cpp_files} PROPERTIES
36	COMPILE_FLAGS -Wno-unused)	48	COMPILE_FLAGS -Wno-unused)
37	endif()	49	endif()
38	endfunction()	50	endfunction()
39	-- /clang-tools-extra/clangd/quality/CompletionModelCodegen.py	51	++ /clang-tools-extra/clangd/quality/CompletionModelCodegen.py
Lines 1-13 Link Here
1	"""Code generator for Code Completion Model Inference.	1	"""Code generator for Code Completion Model Inference.
2		2
3	Tool runs on the Decision Forest model defined in {model} directory.	3	Tool runs on the Decision Forest model defined in {model} directory.
4	It generates two files: {output_dir}/{filename}.h and {output_dir}/{filename}.cpp	4	It generates files: {output_dir}/{filename}.h, {output_dir}/{filename}.cpp,
5	The generated files defines the Example class named {cpp_class} having all the features as class members.	5	and {output_dir}/{filename}{number}.cpp for each Decision Tree.
		6	The generated files define the Example class named {cpp_class} having all the features as class members.
6	The generated runtime provides an `Evaluate` function which can be used to score a code completion candidate.	7	The generated runtime provides an `Evaluate` function which can be used to score a code completion candidate.
7	"""	8	"""
8		9
9	import argparse	10	import argparse
10	import json	11	import json
		12	import os
11	import struct	13	import struct
12		14
13		15
Lines 194-227 Link Here
194	`float Evaluate(const {Example}&)` function. This function can be	196	`float Evaluate(const {Example}&)` function. This function can be
195	used to score an Example."""	197	used to score an Example."""
196		198
197	code = ""	199	functions_codes = {}
198		200
199	# Generate evaluation function of each tree.	201	# Generate evaluation function of each tree.
200	code += "namespace {\n"
201	tree_num = 0	202	tree_num = 0
202	for tree_json in forest_json:	203	for tree_json in forest_json:
203	code += "LLVM_ATTRIBUTE_NOINLINE float EvaluateTree%d(const %s& E) {\n" % (tree_num, cpp_class.name)	204	functions_codes[f"{tree_num}"] = "LLVM_ATTRIBUTE_NOINLINE float EvaluateTree%d(const %s& E) {\n" % (tree_num, cpp_class.name)
204	code += " " + \	205	functions_codes[f"{tree_num}"] += " " + \
205	"\n ".join(	206	"\n ".join(
206	tree(tree_json, tree_num=tree_num, node_num=0)[0]) + "\n"	207	tree(tree_json, tree_num=tree_num, node_num=0)[0]) + "\n"
207	code += "}\n\n"	208	functions_codes[f"{tree_num}"] += "}\n"
208	tree_num += 1	209	tree_num += 1
209	code += "} // namespace\n\n"
210		210
211	# Combine the scores of all trees in the final function.	211	# Combine the scores of all trees in the final function.
212	# MSAN will timeout if these functions are inlined.	212	# MSAN will timeout if these functions are inlined.
213	code += "float Evaluate(const %s& E) {\n" % cpp_class.name	213	final_function_code = ""
214	code += " float Score = 0;\n"
215	for tree_num in range(len(forest_json)):	214	for tree_num in range(len(forest_json)):
216	code += " Score += EvaluateTree%d(E);\n" % tree_num	215	final_function_code += "float EvaluateTree%d(const %s& E);\n" % (tree_num, cpp_class.name)
217	code += " return Score;\n"	216	final_function_code += "\n"
218	code += "}\n"	217	final_function_code += "float Evaluate(const %s& E) {\n" % cpp_class.name
		218	final_function_code += " float Score = 0;\n"
		219	for tree_num in range(len(forest_json)):
		220	final_function_code += " Score += EvaluateTree%d(E);\n" % tree_num
		221	final_function_code += " return Score;\n"
		222	final_function_code += "}\n"
219		223
220	return code	224	return functions_codes, final_function_code
221		225
222		226
223	def gen_cpp_code(forest_json, features_json, filename, cpp_class):	227	def gen_cpp_code(forest_json, features_json, filename, cpp_class):
224	"""Generates code for the .cpp file."""	228	"""Generates code for the .cpp files."""
225	# Headers	229	# Headers
226	# Required by OrderEncode(float F).	230	# Required by OrderEncode(float F).
227	angled_include = [	231	angled_include = [
Lines 242-252 Link Here
242	for feature in features_json	246	for feature in features_json
243	if feature["kind"] == "ENUM")	247	if feature["kind"] == "ENUM")
244	nl = "\n"	248	nl = "\n"
245	return """%s
246		249
247	%s	250	functions_codes, final_function_code = evaluate_func(forest_json, cpp_class)
248		251
249	#define BIT(X) (1LL << X)	252	cpp_code = {
		253	"": """%s
250		254
251	%s	255	%s
252		256
Lines 268-275 Link Here
268	%s	272	%s
269	%s	273	%s
270	""" % (nl.join(angled_include), nl.join(quoted_include), cpp_class.ns_begin(),	274	""" % (nl.join(angled_include), nl.join(quoted_include), cpp_class.ns_begin(),
271	using_decls, cpp_class.name, evaluate_func(forest_json, cpp_class),	275	cpp_class.name, final_function_code, cpp_class.ns_end())}
272	cpp_class.ns_end())	276
		277	for function_code_num, function_code in functions_codes.items():
		278	cpp_code[function_code_num] = """%s
		279
		280	#define BIT(X) (1LL << X)
		281
		282	%s
		283
		284	%s
		285
		286	%s
		287	%s
		288	""" % (nl.join(quoted_include), cpp_class.ns_begin(), using_decls,
		289	function_code, cpp_class.ns_end())
		290
		291	return cpp_code
273		292
274		293
275	def main():	294	def main():
Lines 286-292 Link Here
286	output_dir = ns.output_dir	305	output_dir = ns.output_dir
287	filename = ns.filename	306	filename = ns.filename
288	header_file = "%s/%s.h" % (output_dir, filename)	307	header_file = "%s/%s.h" % (output_dir, filename)
289	cpp_file = "%s/%s.cpp" % (output_dir, filename)
290	cpp_class = CppClass(cpp_class=ns.cpp_class)	308	cpp_class = CppClass(cpp_class=ns.cpp_class)
291		309
292	model_file = "%s/forest.json" % ns.model	310	model_file = "%s/forest.json" % ns.model
Lines 298-309 Link Here
298	with open(model_file) as m:	316	with open(model_file) as m:
299	forest_json = json.load(m)	317	forest_json = json.load(m)
300		318
301	with open(cpp_file, 'w+t') as output_cc:	319	os.makedirs(output_dir, exist_ok=True)
302	output_cc.write(	320
303	gen_cpp_code(forest_json=forest_json,	321	for cpp_code_num, cpp_code in gen_cpp_code(forest_json=forest_json,
304	features_json=features_json,	322	features_json=features_json,
305	filename=filename,	323	filename=filename,
306	cpp_class=cpp_class))	324	cpp_class=cpp_class).items():
		325	cpp_file = "%s/%s%s.cpp" % (output_dir, filename, cpp_code_num)
		326	with open(cpp_file, 'w+t') as output_cc:
		327	output_cc.write(cpp_code)
307		328
308	with open(header_file, 'w+t') as output_h:	329	with open(header_file, 'w+t') as output_h:
309	output_h.write(gen_header_code(	330	output_h.write(gen_header_code(
310	-- /clang-tools-extra/clangd/unittests/CMakeLists.txt	331	++ /clang-tools-extra/clangd/unittests/CMakeLists.txt
Lines 21-26 Link Here
21		21
22	include(${CMAKE_CURRENT_SOURCE_DIR}/../quality/CompletionModel.cmake)	22	include(${CMAKE_CURRENT_SOURCE_DIR}/../quality/CompletionModel.cmake)
23	gen_decision_forest(${CMAKE_CURRENT_SOURCE_DIR}/decision_forest_model DecisionForestRuntimeTest ::ns1::ns2::test::Example)	23	gen_decision_forest(${CMAKE_CURRENT_SOURCE_DIR}/decision_forest_model DecisionForestRuntimeTest ::ns1::ns2::test::Example)
		24	file(GLOB DecisionForestRuntimeTest_files "${CMAKE_CURRENT_BINARY_DIR}/DecisionForestRuntimeTest/DecisionForestRuntimeTest*.cpp")
24		25
25	add_custom_target(ClangdUnitTests)	26	add_custom_target(ClangdUnitTests)
26	add_unittest(ClangdUnitTests ClangdTests	27	add_unittest(ClangdUnitTests ClangdTests
Lines 95-101 Link Here
95	TypeHierarchyTests.cpp	96	TypeHierarchyTests.cpp
96	URITests.cpp	97	URITests.cpp
97	XRefsTests.cpp	98	XRefsTests.cpp
98	${CMAKE_CURRENT_BINARY_DIR}/DecisionForestRuntimeTest.cpp	99	${DecisionForestRuntimeTest_files}
99		100
100	support/CancellationTests.cpp	101	support/CancellationTests.cpp
101	support/ContextTests.cpp	102	support/ContextTests.cpp
Lines 134-142 Link Here
134	$<TARGET_OBJECTS:obj.clangDaemonTweaks>	135	$<TARGET_OBJECTS:obj.clangDaemonTweaks>
135	)	136	)
136		137
137	# Include generated ComletionModel headers.	138	# Include generated Completion Model header.
138	target_include_directories(ClangdTests PUBLIC	139	target_include_directories(ClangdTests PUBLIC
139	$<BUILD_INTERFACE:${CMAKE_CURRENT_BINARY_DIR}>	140	$<BUILD_INTERFACE:${CMAKE_CURRENT_BINARY_DIR}/DecisionForestRuntimeTest>
140	)	141	)
141		142
142	clang_target_link_libraries(ClangdTests	143	clang_target_link_libraries(ClangdTests