More logs

.asf.yaml

@@ -15,14 +15,19 @@
 # specific language governing permissions and limitations
 # under the License.
 
+# This file controls the settings of this repository
+#
+# See more details at
+# https://cwiki.apache.org/confluence/display/INFRA/Git+-+.asf.yaml+features
+
 notifications:
-  commits:      commits@arrow.apache.org
-  issues:       github@arrow.apache.org
-  pullrequests: github@arrow.apache.org
+  commits: commits@datafusion.apache.org
+  issues: github@datafusion.apache.org
+  pullrequests: github@datafusion.apache.org
   jira_options: link label worklog
 github:
-  description: "Apache Arrow DataFusion SQL Query Engine"
-  homepage: https://arrow.apache.org/datafusion
+  description: "Apache DataFusion SQL Query Engine"
+  homepage: https://datafusion.apache.org/
   labels:
     - arrow
     - big-data
@@ -36,6 +41,15 @@ github:
   enabled_merge_buttons:
     squash: true
     merge: false
-    rebase: false    
+    rebase: false
   features:
-    issues: true        
+    issues: true
+  protected_branches:
+    main:
+      required_pull_request_reviews:
+        required_approving_review_count: 1
+
+# publishes the content of the `asf-site` branch to
+# https://datafusion.apache.org/
+publish:
+  whoami: asf-site

.gitattributes

@@ -0,0 +1,4 @@
+.github/ export-ignore
+datafusion/core/tests/data/newlines_in_values.csv text eol=lf
+datafusion/proto/src/generated/prost.rs linguist-generated
+datafusion/proto/src/generated/pbjson.rs linguist-generated

.github/ISSUE_TEMPLATE/bug_report.yml

@@ -0,0 +1,27 @@
+name: Bug report
+description: Create a report to help us improve
+labels: bug
+body:
+  - type: textarea
+    attributes:
+      label: Describe the bug
+      description: Describe the bug.
+      placeholder: >
+        A clear and concise description of what the bug is.
+    validations:
+      required: true
+  - type: textarea
+    attributes:
+      label: To Reproduce
+      placeholder: >
+        Steps to reproduce the behavior:
+  - type: textarea
+    attributes:
+      label: Expected behavior
+      placeholder: >
+        A clear and concise description of what you expected to happen.
+  - type: textarea
+    attributes:
+      label: Additional context
+      placeholder: >
+        Add any other context about the problem here.

.github/ISSUE_TEMPLATE/feature_request.yml

@@ -0,0 +1,26 @@
+name: Feature request
+description: Suggest an idea for this project
+labels: enhancement
+body:
+  - type: textarea
+    attributes:
+      label: Is your feature request related to a problem or challenge?
+      description: Please describe what you are trying to do.
+      placeholder: >
+        A clear and concise description of what the problem is. Ex. I'm always frustrated when [...]
+        (This section helps Arrow developers understand the context and *why* for this feature, in addition to the *what*)
+  - type: textarea
+    attributes:
+      label: Describe the solution you'd like
+      placeholder: >
+        A clear and concise description of what you want to happen.
+  - type: textarea
+    attributes:
+      label: Describe alternatives you've considered
+      placeholder: >
+        A clear and concise description of any alternative solutions or features you've considered.
+  - type: textarea
+    attributes:
+      label: Additional context
+      placeholder: >
+        Add any other context or screenshots about the feature request here.

.github/actions/setup-builder/action.yaml

@@ -28,12 +28,21 @@ runs:
     - name: Install Build Dependencies
       shell: bash
       run: |
-        apt-get update
-        apt-get install -y protobuf-compiler
+        RETRY="ci/scripts/retry"
+        "${RETRY}" apt-get update
+        "${RETRY}" apt-get install -y protobuf-compiler
     - name: Setup Rust toolchain
       shell: bash
+      # rustfmt is needed for the substrait build script
       run: |
+        RETRY="ci/scripts/retry"
         echo "Installing ${{ inputs.rust-version }}"
-        rustup toolchain install ${{ inputs.rust-version }}
-        rustup default ${{ inputs.rust-version }}
-        rustup component add rustfmt
+        "${RETRY}" rustup toolchain install ${{ inputs.rust-version }}
+        "${RETRY}" rustup default ${{ inputs.rust-version }}
+        "${RETRY}" rustup component add rustfmt
+    - name: Configure rust runtime env
+      uses: ./.github/actions/setup-rust-runtime
+    - name: Fixup git permissions
+      # https://github.com/actions/checkout/issues/766
+      shell: bash
+      run: git config --global --add safe.directory "$GITHUB_WORKSPACE"

.github/actions/setup-macos-aarch64-builder/action.yaml

@@ -0,0 +1,47 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+name: Prepare Rust Builder for MacOS
+description: 'Prepare Rust Build Environment for MacOS'
+inputs:
+  rust-version:
+    description: 'version of rust to install (e.g. stable)'
+    required: true
+    default: 'stable'
+runs:
+  using: "composite"
+  steps:
+    - name: Install protobuf compiler
+      shell: bash
+      run: |
+        mkdir -p $HOME/d/protoc
+        cd $HOME/d/protoc
+        export PROTO_ZIP="protoc-21.4-osx-aarch_64.zip"        
+        curl -LO https://github.com/protocolbuffers/protobuf/releases/download/v21.4/$PROTO_ZIP
+        unzip $PROTO_ZIP
+        echo "$HOME/d/protoc/bin" >> $GITHUB_PATH
+        export PATH=$PATH:$HOME/d/protoc/bin
+        protoc --version
+    - name: Setup Rust toolchain
+      shell: bash
+      run: |
+        rustup update stable
+        rustup toolchain install stable
+        rustup default stable
+        rustup component add rustfmt
+    - name: Configure rust runtime env
+      uses: ./.github/actions/setup-rust-runtime        

.github/actions/setup-macos-builder/action.yaml

@@ -0,0 +1,47 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+name: Prepare Rust Builder for MacOS
+description: 'Prepare Rust Build Environment for MacOS'
+inputs:
+  rust-version:
+    description: 'version of rust to install (e.g. stable)'
+    required: true
+    default: 'stable'
+runs:
+  using: "composite"
+  steps:
+    - name: Install protobuf compiler
+      shell: bash
+      run: |
+        mkdir -p $HOME/d/protoc
+        cd $HOME/d/protoc
+        export PROTO_ZIP="protoc-21.4-osx-x86_64.zip"        
+        curl -LO https://github.com/protocolbuffers/protobuf/releases/download/v21.4/$PROTO_ZIP
+        unzip $PROTO_ZIP
+        echo "$HOME/d/protoc/bin" >> $GITHUB_PATH
+        export PATH=$PATH:$HOME/d/protoc/bin
+        protoc --version
+    - name: Setup Rust toolchain
+      shell: bash
+      run: |
+        rustup update stable
+        rustup toolchain install stable
+        rustup default stable
+        rustup component add rustfmt
+    - name: Configure rust runtime env
+      uses: ./.github/actions/setup-rust-runtime        

.github/actions/setup-rust-runtime/action.yaml

@@ -0,0 +1,38 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+name: Setup Rust Runtime
+description: 'Setup Rust Runtime Environment'
+runs:
+  using: "composite"
+  steps:
+    - name: Run sccache-cache
+      uses: mozilla-actions/[email protected]
+    - name: Configure runtime env
+      shell: bash
+      # do not produce debug symbols to keep memory usage down
+      # hardcoding other profile params to avoid profile override values
+      # More on Cargo profiles https://doc.rust-lang.org/cargo/reference/profiles.html?profile-settings#profile-settings
+      # 
+      # Set debuginfo=line-tables-only as debuginfo=0 causes immensely slow build
+      # See for more details: https://github.com/rust-lang/rust/issues/119560
+      run: |
+        echo "RUSTC_WRAPPER=sccache" >> $GITHUB_ENV  
+        echo "SCCACHE_GHA_ENABLED=true" >> $GITHUB_ENV 
+        echo "RUST_BACKTRACE=1" >> $GITHUB_ENV
+        echo "RUSTFLAGS=-C debuginfo=line-tables-only -C incremental=false" >> $GITHUB_ENV
+

.github/actions/setup-windows-builder/action.yaml

@@ -0,0 +1,46 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+name: Prepare Rust Builder for Windows
+description: 'Prepare Rust Build Environment for Windows'
+inputs:
+  rust-version:
+    description: 'version of rust to install (e.g. stable)'
+    required: true
+    default: 'stable'
+runs:
+  using: "composite"
+  steps:
+    - name: Install protobuf compiler
+      shell: bash
+      run: |
+        mkdir -p $HOME/d/protoc
+        cd $HOME/d/protoc
+        export PROTO_ZIP="protoc-21.4-win64.zip"
+        curl -LO https://github.com/protocolbuffers/protobuf/releases/download/v21.4/$PROTO_ZIP
+        unzip $PROTO_ZIP
+        export PATH=$PATH:$HOME/d/protoc/bin
+        protoc.exe --version
+    - name: Setup Rust toolchain
+      shell: bash
+      run: |
+        # Avoid self update to avoid CI failures: https://github.com/apache/datafusion/issues/9653
+        rustup toolchain install stable --no-self-update
+        rustup default stable
+        rustup component add rustfmt
+    - name: Configure rust runtime env
+      uses: ./.github/actions/setup-rust-runtime        

.github/dependabot.yml

@@ -22,8 +22,26 @@ updates:
     schedule:
       interval: daily
     open-pull-requests-limit: 10
-    target-branch: master
+    target-branch: main
     labels: [auto-dependencies]
+    ignore:
+      # arrow is bumped manually
+      - dependency-name: "arrow*"
+        update-types: ["version-update:semver-major"]
+  - package-ecosystem: cargo
+    directory: "datafusion-cli/"
+    schedule:
+      interval: daily
+    open-pull-requests-limit: 10
+    target-branch: main
+    labels: [auto-dependencies]
+    ignore:
+      # arrow is bumped manually
+      - dependency-name: "arrow*"
+        update-types: ["version-update:semver-major"]
+      # datafusion is bumped manually
+      - dependency-name: "datafusion*"
+        update-types: ["version-update:semver-major"]
   - package-ecosystem: "github-actions"
     directory: "/"
     schedule: