index.html

<!DOCTYPE html>


<html lang="zh-CN">
<head>
  <meta charset="UTF-8">
<meta name="viewport" content="width=device-width, initial-scale=1, maximum-scale=2">
<meta name="theme-color" content="#222">
<meta name="generator" content="Hexo 3.9.0">
  <link rel="apple-touch-icon" sizes="180x180" href="/images/apple-touch-icon-next.png?v=7.4.1">
  <link rel="icon" type="image/png" sizes="32x32" href="/images/favicon-32x32-next.png?v=7.4.1">
  <link rel="icon" type="image/png" sizes="16x16" href="/images/favicon-16x16-next.png?v=7.4.1">
  <link rel="mask-icon" href="/images/logo.svg?v=7.4.1" color="#222">

<link rel="stylesheet" href="/css/main.css?v=7.4.1">


<link rel="stylesheet" href="/lib/font-awesome/css/font-awesome.min.css?v=4.7.0">


<script id="hexo-configurations">
  var NexT = window.NexT || {};
  var CONFIG = {
    root: '/',
    scheme: 'Mist',
    version: '7.4.1',
    exturl: false,
    sidebar: {"position":"right","display":"hide","offset":12,"onmobile":false},
    copycode: {"enable":false,"show_result":false,"style":null},
    back2top: {"enable":true,"sidebar":false,"scrollpercent":false},
    bookmark: {"enable":false,"color":"#222","save":"auto"},
    fancybox: false,
    mediumzoom: false,
    lazyload: false,
    pangu: false,
    algolia: {
      appID: '',
      apiKey: '',
      indexName: '',
      hits: {"per_page":10},
      labels: {"input_placeholder":"Search for Posts","hits_empty":"We didn't find any results for the search: ${query}","hits_stats":"${hits} results found in ${time} ms"}
    },
    localsearch: {"enable":false,"trigger":"auto","top_n_per_article":1,"unescape":false,"preload":false},
    path: '',
    motion: {"enable":true,"async":false,"transition":{"post_block":"fadeIn","post_header":"slideDownIn","post_body":"slideDownIn","coll_header":"slideLeftIn","sidebar":"slideUpIn"}},
    translation: {
      copy_button: '复制',
      copy_success: '复制成功',
      copy_failure: '复制失败'
    },
    sidebarPadding: 40
  };
</script>

  <meta property="og:type" content="website">
<meta property="og:title" content="纪光的博客">
<meta property="og:url" content="http://yoursite.com/index.html">
<meta property="og:site_name" content="纪光的博客">
<meta property="og:locale" content="zh-CN">
<meta name="twitter:card" content="summary">
<meta name="twitter:title" content="纪光的博客">
  <link rel="canonical" href="http://yoursite.com/">


<script id="page-configurations">
  // https://hexo.io/docs/variables.html
  CONFIG.page = {
    sidebar: "",
    isHome: true,
    isPost: false,
    isPage: false,
    isArchive: false
  };
</script>

  <title>纪光的博客</title>
  

  <script>
    var _hmt = _hmt || [];
    (function() {
      var hm = document.createElement("script");
      hm.src = "https://hm.baidu.com/hm.js?bca403b673d79e83a47dc62fea1dd605";
      var s = document.getElementsByTagName("script")[0];
      s.parentNode.insertBefore(hm, s);
    })();
  </script>


  <noscript>
  <style>
  .use-motion .brand,
  .use-motion .menu-item,
  .sidebar-inner,
  .use-motion .post-block,
  .use-motion .pagination,
  .use-motion .comments,
  .use-motion .post-header,
  .use-motion .post-body,
  .use-motion .collection-header { opacity: initial; }

  .use-motion .site-title,
  .use-motion .site-subtitle {
    opacity: initial;
    top: initial;
  }

  .use-motion .logo-line-before i { left: initial; }
  .use-motion .logo-line-after i { right: initial; }
  </style>
</noscript>

</head>

<body itemscope itemtype="http://schema.org/WebPage">
  <div class="container use-motion">
    <div class="headband"></div>

    <header class="header" itemscope itemtype="http://schema.org/WPHeader">
      <div class="header-inner"><div class="site-brand-container">
  <div class="site-meta">

    <div>
      <a href="/" class="brand" rel="start">
        <span class="logo-line-before"><i></i></span>
        <span class="site-title">纪光的博客</span>
        <span class="logo-line-after"><i></i></span>
      </a>
    </div>
  </div>

  <div class="site-nav-toggle">
    <div class="toggle" aria-label="切换导航栏">
      <span class="toggle-line toggle-line-first"></span>
      <span class="toggle-line toggle-line-middle"></span>
      <span class="toggle-line toggle-line-last"></span>
    </div>
  </div>
</div>


<nav class="site-nav">
  
  <ul id="menu" class="menu">
      
      
        <li class="menu-item menu-item-home">
      
    
    <a href="/" rel="section"><i class="fa fa-fw fa-home"></i>首页</a>

  </li>
      
    
        <li class="menu-item menu-item-categories">
      
    
    <a href="/categories/" rel="section"><i class="fa fa-fw fa-th"></i>分类</a>

  </li>
      
    
        <li class="menu-item menu-item-tags">
      
    
    <a href="/tags/" rel="section"><i class="fa fa-fw fa-tags"></i>标签</a>

  </li>
      
    
        <li class="menu-item menu-item-archives">
      
    
    <a href="/archives/" rel="section"><i class="fa fa-fw fa-archive"></i>归档</a>

  </li>
      
    
        <li class="menu-item menu-item-about">
      
    
    <a href="/about/" rel="section"><i class="fa fa-fw fa-user"></i>关于</a>

  </li>
      
    
  </ul>

    
</nav>
</div>
    </header>

    
  <div class="back-to-top">
    <i class="fa fa-arrow-up"></i>
    <span>0%</span>
  </div>


    <main class="main">
      <div class="main-inner">
        <div class="content-wrap">
          <div class="content">
            

  <div class="posts-expand">
        
  
  <article itemscope itemtype="http://schema.org/Article" class="post-block home" lang="zh-CN">
    <link itemprop="mainEntityOfPage" href="http://yoursite.com/2021/05/31/clickhouse/ClickHouse基本知识/">

    <span hidden itemprop="author" itemscope itemtype="http://schema.org/Person">
      <meta itemprop="name" content="姜纪光">
      <meta itemprop="description" content="">
      <meta itemprop="image" content="/images/avatar.gif">
    </span>

    <span hidden itemprop="publisher" itemscope itemtype="http://schema.org/Organization">
      <meta itemprop="name" content="纪光的博客">
    </span>
      <header class="post-header">
        <h1 class="post-title" itemprop="name headline">
          
            
            <a href="/2021/05/31/clickhouse/ClickHouse基本知识/" class="post-title-link" itemprop="url">ClickHouse基本知识</a>
          
        </h1>

        <div class="post-meta">
            <span class="post-meta-item">
              <span class="post-meta-item-icon">
                <i class="fa fa-calendar-o"></i>
              </span>
              <span class="post-meta-item-text">发表于</span>

              
              <time title="创建时间：2021-05-31 08:55:13" itemprop="dateCreated datePublished" datetime="2021-05-31T08:55:13+08:00">2021-05-31</time>
            </span>
          
            
              <span class="post-meta-item">
                <span class="post-meta-item-icon">
                  <i class="fa fa-calendar-check-o"></i>
                </span>
                <span class="post-meta-item-text">更新于</span>
                <time title="修改时间：2021-06-20 12:19:35" itemprop="dateModified" datetime="2021-06-20T12:19:35+08:00">2021-06-20</time>
              </span>
            
          
            <span class="post-meta-item">
              <span class="post-meta-item-icon">
                <i class="fa fa-folder-o"></i>
              </span>
              <span class="post-meta-item-text">分类于</span>
              
                <span itemprop="about" itemscope itemtype="http://schema.org/Thing">
                  <a href="/categories/ClickHouse/" itemprop="url" rel="index">
                    <span itemprop="name">ClickHouse</span>
                  </a>
                </span>

                
            </span>
          

            <span id="/2021/05/31/clickhouse/ClickHouse基本知识/" class="post-meta-item leancloud_visitors" data-flag-title="ClickHouse基本知识" title="阅读次数">
              <span class="post-meta-item-icon">
                <i class="fa fa-eye"></i>
              </span>
              <span class="post-meta-item-text">阅读次数：</span>
              <span class="leancloud-visitors-count"></span>
            </span>
          
  
        </div>
      </header>

    
    <div class="post-body" itemprop="articleBody">

      
          <h1 id="基本概念"><a href="#基本概念" class="headerlink" title="基本概念"></a>基本概念</h1><ul>
<li>ck中严格区分大小写</li>
<li>分片，将一张表分成多个部分</li>
<li>副本</li>
<li>分区，对表数据管理优化的手段</li>
<li>表</li>
<li>引擎：不同的引擎，底层的数据存储和维护管理不同，对表的操作行为不一样</li>
<li>向量化</li>
<li></li>
</ul>
<h1 id="常用命令"><a href="#常用命令" class="headerlink" title="常用命令"></a>常用命令</h1><h1 id="基本语法"><a href="#基本语法" class="headerlink" title="基本语法"></a>基本语法</h1><figure class="highlight sql"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br></pre></td><td class="code"><pre><span class="line"><span class="comment">-- show databases;</span></span><br><span class="line"><span class="comment">-- create database if not exists test1;</span></span><br><span class="line"><span class="comment">-- use test1;</span></span><br><span class="line"><span class="comment">-- select currentDatabase();</span></span><br><span class="line"><span class="comment">-- drop database test1;</span></span><br></pre></td></tr></table></figure>

<h2 id="DDL"><a href="#DDL" class="headerlink" title="DDL"></a>DDL</h2><h1 id="优点和缺点"><a href="#优点和缺点" class="headerlink" title="优点和缺点"></a>优点和缺点</h1><ul>
<li><p>优点</p>
<blockquote>
<p>1、</p>
</blockquote>
</li>
<li><p>缺点</p>
<blockquote>
<p>1、不支持事务，不支持真正的删除/更新</p>
<p>2、不支持高并发，官方建议qps 100，在服务器足够好的情况下，可以通过配置文件修改连接数</p>
<p>3、不支持二级索引</p>
<p>4、不擅长夺标join</p>
<p>5、元数据管理需要人为干预</p>
<p>6、尽量做1000条以上的批量写入</p>
<p>7、zookeeper依赖</p>
</blockquote>
</li>
</ul>
<h1 id="数据类型"><a href="#数据类型" class="headerlink" title="数据类型"></a>数据类型</h1><figure class="highlight sql"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br></pre></td><td class="code"><pre><span class="line"><span class="comment">-- ck中建表时，一定指定表引擎</span></span><br><span class="line"></span><br><span class="line"><span class="keyword">create</span> <span class="keyword">table</span> tb_test1 (</span><br><span class="line">  <span class="keyword">id</span> Int16, </span><br><span class="line">  <span class="keyword">name</span> <span class="keyword">String</span></span><br><span class="line">)<span class="keyword">engine</span>=<span class="keyword">Memory</span>;</span><br></pre></td></tr></table></figure>

<figure class="highlight sql"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br></pre></td><td class="code"><pre><span class="line"></span><br><span class="line"></span><br></pre></td></tr></table></figure>

<h1 id="视图"><a href="#视图" class="headerlink" title="视图"></a>视图</h1><ul>
<li>分为普通和物化两种视图</li>
<li>普通视图：只是一层简单的查询代理，对性能不会有任何增强</li>
<li>物化视图：拥有独立的存储，当原表插入数据时会自动同步到物化视图；删除原表数据，不会删除物化视图的数据</li>
</ul>
<h1 id="引擎"><a href="#引擎" class="headerlink" title="引擎"></a>引擎</h1><h2 id="Log"><a href="#Log" class="headerlink" title="Log"></a>Log</h2><ul>
<li>最小功能引擎</li>
<li>适用场景：快速写入许多小表（最多约100万行）并在以后整体读取该表时</li>
</ul>
<h3 id="TinyLog"><a href="#TinyLog" class="headerlink" title="TinyLog"></a>TinyLog</h3><ul>
<li>最简单的表引擎，用于将数据保存在磁盘上。</li>
<li>每列都存储在单独的压缩文件中，写入时数据附加到文件末尾</li>
<li>没有索引，没有标记块</li>
<li>不允许同时读写：insert into t select * from t1</li>
</ul>
<h3 id="Stripelog"><a href="#Stripelog" class="headerlink" title="Stripelog"></a>Stripelog</h3><ul>
<li>data.bin存储所有数据</li>
<li>index.mrk对数据建立索引</li>
<li>size.json数据大小</li>
<li>并发读写</li>
</ul>
<h3 id="Log-1"><a href="#Log-1" class="headerlink" title="Log"></a>Log</h3><ul>
<li>*.bin存储每个字段的数据</li>
<li>mark.mrk 数据块标记</li>
<li>支持多线程处理</li>
<li>并发读写</li>
</ul>
<h2 id="MergeTree"><a href="#MergeTree" class="headerlink" title="MergeTree"></a>MergeTree</h2><ul>
<li>只有MergeTree的引擎才支持表结构更改</li>
<li>一定要指定主键和排序字段</li>
<li>查看建表语句，就可以查看引擎参数等</li>
<li>只有MergeTree引擎的数据才能修改</li>
<li>只有MergeTree引擎支持分区</li>
<li>支持数据复制和采样</li>
</ul>
<h3 id="MergeTree-1"><a href="#MergeTree-1" class="headerlink" title="MergeTree"></a>MergeTree</h3><ul>
<li>没有去重功能</li>
</ul>
<h3 id="ReplacingMergeTree"><a href="#ReplacingMergeTree" class="headerlink" title="ReplacingMergeTree"></a>ReplacingMergeTree</h3><ul>
<li><p>使用order by 排序字段作为判断重复数据的唯一依据</p>
</li>
<li><p>只有的在合并分区时才会触发删除重复数据的逻辑</p>
</li>
<li><p>以分区为单位删除重复数据</p>
</li>
<li><p>去重策略</p>
<blockquote>
<p>1、如果没有设置版本号，那么保留最后一行</p>
<p>2、如果设置了版本号，那么保留版本字段中最大的一行</p>
</blockquote>
</li>
</ul>
<h2 id="CollapsingMergeTree"><a href="#CollapsingMergeTree" class="headerlink" title="CollapsingMergeTree"></a>CollapsingMergeTree</h2><ul>
<li>乱序写入无法删除</li>
</ul>
<h2 id="VersionedCollapsingMergeTree"><a href="#VersionedCollapsingMergeTree" class="headerlink" title="VersionedCollapsingMergeTree"></a>VersionedCollapsingMergeTree</h2><ul>
<li>排序字段相同，version相同，且sign相反的行，在合并时会删除</li>
</ul>
<h2 id="SummingMergeTree"><a href="#SummingMergeTree" class="headerlink" title="SummingMergeTree"></a>SummingMergeTree</h2><ul>
<li>终端用户只关心查询数据的汇总结果，不关心名字数据</li>
</ul>
<h2 id="AggregatingMergeTree"><a href="#AggregatingMergeTree" class="headerlink" title="AggregatingMergeTree"></a>AggregatingMergeTree</h2><ul>
<li>在物化视图下，可以做到同步聚合</li>
</ul>
<h2 id="外部存储引擎"><a href="#外部存储引擎" class="headerlink" title="外部存储引擎"></a>外部存储引擎</h2><h3 id="HDFS引擎"><a href="#HDFS引擎" class="headerlink" title="HDFS引擎"></a>HDFS引擎</h3><ul>
<li>clickhouse可以直接从HDFS指定目录下加载数据，自己不存储数据，仅仅是读取数据</li>
<li>数据如果已经存在，则不能再插入</li>
</ul>
<h3 id="mysql引擎"><a href="#mysql引擎" class="headerlink" title="mysql引擎"></a>mysql引擎</h3><ul>
<li><h2 id="内存引擎"><a href="#内存引擎" class="headerlink" title="内存引擎"></a>内存引擎</h2></li>
</ul>
<h3 id="Memory"><a href="#Memory" class="headerlink" title="Memory"></a>Memory</h3><h3 id="Buffer"><a href="#Buffer" class="headerlink" title="Buffer"></a>Buffer</h3><ul>
<li><p>完全使用内存装在数据，不支持文件的持久化存储</p>
</li>
<li><p>不是为了面向查询场景而设计的，而是充当缓冲区的角色</p>
<blockquote>
<p>由于写入的并发数很高，可能会导致合并的速度慢于写入速度</p>
</blockquote>
</li>
<li></li>
</ul>
<h1 id="元数据"><a href="#元数据" class="headerlink" title="元数据"></a>元数据</h1><h1 id="目录"><a href="#目录" class="headerlink" title="目录"></a>目录</h1><p>/etc/</p>
<p>/var/lib</p>
<p>/var/log</p>
<h1 id="客户端参数"><a href="#客户端参数" class="headerlink" title="客户端参数"></a>客户端参数</h1><h1 id="端口"><a href="#端口" class="headerlink" title="端口"></a>端口</h1><ul>
<li>9000：client连接server的端口号</li>
<li>8123：通过jdbc协议连接</li>
</ul>
<h1 id="分布式"><a href="#分布式" class="headerlink" title="分布式"></a>分布式</h1><ul>
<li>副本的名称和主机名一致</li>
<li></li>
</ul>

        
    </div>

    
      <footer class="post-footer">
        <div class="post-eof"></div>
      </footer>
  </article>
  
  
  <article itemscope itemtype="http://schema.org/Article" class="post-block home" lang="zh-CN">
    <link itemprop="mainEntityOfPage" href="http://yoursite.com/2021/05/07/flink/Flink运行时架构/">

    <span hidden itemprop="author" itemscope itemtype="http://schema.org/Person">
      <meta itemprop="name" content="姜纪光">
      <meta itemprop="description" content="">
      <meta itemprop="image" content="/images/avatar.gif">
    </span>

    <span hidden itemprop="publisher" itemscope itemtype="http://schema.org/Organization">
      <meta itemprop="name" content="纪光的博客">
    </span>
      <header class="post-header">
        <h1 class="post-title" itemprop="name headline">
          
            
            <a href="/2021/05/07/flink/Flink运行时架构/" class="post-title-link" itemprop="url">Flink运行时架构</a>
          
        </h1>

        <div class="post-meta">
            <span class="post-meta-item">
              <span class="post-meta-item-icon">
                <i class="fa fa-calendar-o"></i>
              </span>
              <span class="post-meta-item-text">发表于</span>

              
              <time title="创建时间：2021-05-07 09:09:07" itemprop="dateCreated datePublished" datetime="2021-05-07T09:09:07+08:00">2021-05-07</time>
            </span>
          
            
              <span class="post-meta-item">
                <span class="post-meta-item-icon">
                  <i class="fa fa-calendar-check-o"></i>
                </span>
                <span class="post-meta-item-text">更新于</span>
                <time title="修改时间：2021-06-20 12:19:35" itemprop="dateModified" datetime="2021-06-20T12:19:35+08:00">2021-06-20</time>
              </span>
            
          
            <span class="post-meta-item">
              <span class="post-meta-item-icon">
                <i class="fa fa-folder-o"></i>
              </span>
              <span class="post-meta-item-text">分类于</span>
              
                <span itemprop="about" itemscope itemtype="http://schema.org/Thing">
                  <a href="/categories/flink/" itemprop="url" rel="index">
                    <span itemprop="name">flink</span>
                  </a>
                </span>

                
            </span>
          

            <span id="/2021/05/07/flink/Flink运行时架构/" class="post-meta-item leancloud_visitors" data-flag-title="Flink运行时架构" title="阅读次数">
              <span class="post-meta-item-icon">
                <i class="fa fa-eye"></i>
              </span>
              <span class="post-meta-item-text">阅读次数：</span>
              <span class="leancloud-visitors-count"></span>
            </span>
          
  
        </div>
      </header>

    
    <div class="post-body" itemprop="articleBody">

      
          <h1 id="运行时组件"><a href="#运行时组件" class="headerlink" title="运行时组件"></a>运行时组件</h1><p><img src="/images/flink/flink_submit.png" alt></p>
<p><img src="/images/flink/flink_submit_yarn.png" alt></p>
<h2 id="作业管理器（JobManager）"><a href="#作业管理器（JobManager）" class="headerlink" title="作业管理器（JobManager）"></a>作业管理器（JobManager）</h2><ul>
<li>控制一个应用程序执行的主进程</li>
<li>jobmanager会先收到应用程序，包括 JobGraph、逻辑数据流图（logical dataflow graph）、所有的类库和jar包等</li>
<li>jobmanager会把JobGraph转成ExcutionGraph（物理层面的数据流图），包含了所有可以并发执行的任务</li>
<li>jobmanager向ResourceManager请求资源（TaskManager），然后将ExcutionGraph分发到TaskManager上执行</li>
<li>在作业运行过程中，jobmanager负责所有需要中央协调的操作，比如checkpoint的协调</li>
</ul>
<h2 id="任务管理器（TaskManager）"><a href="#任务管理器（TaskManager）" class="headerlink" title="任务管理器（TaskManager）"></a>任务管理器（TaskManager）</h2><ul>
<li>flink中的工作进程，每个TaskManager都有一定数量的slot</li>
<li>TaskManager启动之后，会向资源管理器注册它的slot，然后JobManager通过资源管理器可以使用这些slot，来执行task</li>
<li>子task slot共享</li>
</ul>
<h2 id="资源管理器（ResourceManager）"><a href="#资源管理器（ResourceManager）" class="headerlink" title="资源管理器（ResourceManager）"></a>资源管理器（ResourceManager）</h2><ul>
<li>不是YARN的ResourceManager，而是ApplicationMaster中的ResourceManager</li>
<li>主要负责管理TaskManager中的slot</li>
<li>flink为不同的环境和资源管理工具提供了不同的资源管理器</li>
</ul>
<h2 id="分发器（Dispacher）"><a href="#分发器（Dispacher）" class="headerlink" title="分发器（Dispacher）"></a>分发器（Dispacher）</h2><ul>
<li>可以跨作业运行，它为应用的提交提供了REST接口</li>
<li>当一个作业被提交时，分发器就会启动并将作业移交给一个JobManager</li>
<li>分发器也会启动一个WEB UI，用以展示和监控作业的执行信息</li>
<li>分发器在架构中不是必须的，取决于作业提交的方式</li>
</ul>
<h1 id="执行图"><a href="#执行图" class="headerlink" title="执行图"></a>执行图</h1><ul>
<li>可以分为4层：StreamGraph -&gt; JobGraph -&gt; ExcecutionGraph -&gt; 物理执行图</li>
<li>StreamGraph：根据作业中的Stream API编写的代码生成的最初的图，表示作业的拓扑结构</li>
<li>JobGraph：StreamGraph经过chain优化后生成JobGraph，提交给JobManager的数据结构</li>
<li>ExcecutionGraph：JobManager将JobGraph并行化生成ExcecutionGraph，ExcecutionGraph是调度层最核心的数据结构</li>
<li>物理执行图：JobManager根据ExcecutionGraph对Job进行调度后，在各个TaskManager部署Task后形成的图，并不是一个具体的数据结构</li>
</ul>

        
    </div>

    
      <footer class="post-footer">
        <div class="post-eof"></div>
      </footer>
  </article>
  
  
  <article itemscope itemtype="http://schema.org/Article" class="post-block home" lang="zh-CN">
    <link itemprop="mainEntityOfPage" href="http://yoursite.com/2021/05/01/flink/Flink关键接口探索/">

    <span hidden itemprop="author" itemscope itemtype="http://schema.org/Person">
      <meta itemprop="name" content="姜纪光">
      <meta itemprop="description" content="">
      <meta itemprop="image" content="/images/avatar.gif">
    </span>

    <span hidden itemprop="publisher" itemscope itemtype="http://schema.org/Organization">
      <meta itemprop="name" content="纪光的博客">
    </span>
      <header class="post-header">
        <h1 class="post-title" itemprop="name headline">
          
            
            <a href="/2021/05/01/flink/Flink关键接口探索/" class="post-title-link" itemprop="url">Flink关键接口探索</a>
          
        </h1>

        <div class="post-meta">
            <span class="post-meta-item">
              <span class="post-meta-item-icon">
                <i class="fa fa-calendar-o"></i>
              </span>
              <span class="post-meta-item-text">发表于</span>

              
              <time title="创建时间：2021-05-01 17:42:00" itemprop="dateCreated datePublished" datetime="2021-05-01T17:42:00+08:00">2021-05-01</time>
            </span>
          
            
              <span class="post-meta-item">
                <span class="post-meta-item-icon">
                  <i class="fa fa-calendar-check-o"></i>
                </span>
                <span class="post-meta-item-text">更新于</span>
                <time title="修改时间：2021-06-20 12:19:35" itemprop="dateModified" datetime="2021-06-20T12:19:35+08:00">2021-06-20</time>
              </span>
            
          
            <span class="post-meta-item">
              <span class="post-meta-item-icon">
                <i class="fa fa-folder-o"></i>
              </span>
              <span class="post-meta-item-text">分类于</span>
              
                <span itemprop="about" itemscope itemtype="http://schema.org/Thing">
                  <a href="/categories/flink/" itemprop="url" rel="index">
                    <span itemprop="name">flink</span>
                  </a>
                </span>

                
            </span>
          

            <span id="/2021/05/01/flink/Flink关键接口探索/" class="post-meta-item leancloud_visitors" data-flag-title="Flink关键接口探索" title="阅读次数">
              <span class="post-meta-item-icon">
                <i class="fa fa-eye"></i>
              </span>
              <span class="post-meta-item-text">阅读次数：</span>
              <span class="leancloud-visitors-count"></span>
            </span>
          
  
        </div>
      </header>

    
    <div class="post-body" itemprop="articleBody">

      
          <h1 id="状态"><a href="#状态" class="headerlink" title="状态"></a>状态</h1><h2 id="CheckpointedFunction"><a href="#CheckpointedFunction" class="headerlink" title="CheckpointedFunction"></a>CheckpointedFunction</h2><ul>
<li><p>状态转换函数的核心接口，该函数持有跨数据元素的状态</p>
</li>
<li><p>该接口提供了针对keyed state 和 operator state 的管理的极大便利</p>
</li>
<li><p>该接口包含两个方法：initializeState  snapshotState</p>
</li>
<li><p>initializeState</p>
<blockquote>
<p>1、当实现该接口的类被实例化时，调用该方法</p>
</blockquote>
</li>
<li><p>snapshotState</p>
<blockquote>
<p>1、每当checkpoint 做该函数的状态快照的时候执行</p>
<p>2、该函数可以作为与外部系统flush/commit/synchronize 的hook</p>
</blockquote>
</li>
<li><p>kafka consumer使用案例</p>
<blockquote>
<p>initializeState：用来获取状态中的每个 partition的 offset</p>
<p>snapshotState：获取每个checkpointId的每个 partition的 offset</p>
</blockquote>
</li>
<li><p>Kafka producer 使用案列</p>
<blockquote>
</blockquote>
</li>
<li><p>TwoPhaseCommitSinkFunction 使用案例</p>
<blockquote>
<p>1、所有实现仅一次语义SinkFunction的建议基类</p>
<p>2、它通过 CheckpointedFunction 和 CheckpointListener 实现两阶段提交算法</p>
<p>3、用户应该提供自定义的事务处理</p>
<p>4、initializeState：</p>
<p>5、snapshotState：两阶段提交中的预提交</p>
</blockquote>
</li>
<li><p>hudi使用案例</p>
<blockquote>
<p>initializeState：从instantState中，获取时间轴上最新已经被处理的时间 issuedInstant</p>
<p>snapshotState：把issuedInstant 保存到 instantState中</p>
</blockquote>
</li>
</ul>
<h2 id="CheckpointListener"><a href="#CheckpointListener" class="headerlink" title="CheckpointListener"></a>CheckpointListener</h2><ul>
<li><p>该接口是被用来与外界进行事务交互，一旦checkpoint完成提交外部事务</p>
</li>
<li><p>Checkpoint Subsuming Contract</p>
<blockquote>
<p>1、有两组artifacts</p>
<p>  ready set：下个checkpoint将要被发布的artifacts</p>
<p>  pending set：当本次checkpoint完成时，发布</p>
</blockquote>
</li>
<li><p>有两个方法：notifyCheckpointComplete 和 notifyCheckpointAborted</p>
</li>
<li><p>notifyCheckpointComplete </p>
<blockquote>
<p>1、通知listener 给定的checkpointId已经完成并且提交</p>
<p>2、”通知行为”是”尽力而为”的，这意味着可能有时会跳过，因此要遵循”Checkpoint Subsuming Contract”</p>
</blockquote>
</li>
<li><p>notifyCheckpointAborted</p>
<blockquote>
<p>1、当分布式的checkpoint被终止时，该方法被调用</p>
<p>2、该方法一般不需要实现</p>
</blockquote>
</li>
<li><p>TwoPhaseCommitSinkFunction使用案例</p>
<blockquote>
</blockquote>
</li>
</ul>
<h1 id="Source"><a href="#Source" class="headerlink" title="Source"></a>Source</h1><h2 id="SourceFunction"><a href="#SourceFunction" class="headerlink" title="SourceFunction"></a>SourceFunction<t></t></h2><ul>
<li><p>所有source算子的基接口</p>
</li>
<li><p>当source开始发射数据时，run方法是被调用的，run方法可以运行足够长的时间，cancel方法用以结束run方法</p>
</li>
<li><p>Timestamps and watermarks</p>
<blockquote>
<p>1、source算子可以给数据指定timestamps，并且可以发射watermarks</p>
<p>2、这在EventTime时才有效，在其它时间类型下，会忽略时间戳和watermarks</p>
</blockquote>
</li>
<li><p>包含两个方法：run 和 cancel</p>
</li>
<li><p>run方法</p>
<blockquote>
<p>启动source，并且使用SourceContext 发射数据</p>
</blockquote>
</li>
<li><p>cancel方法</p>
<blockquote>
<p>一般设置为volatile变量，保证run内部的循环可以结束</p>
</blockquote>
</li>
</ul>
<h3 id="SourceContext"><a href="#SourceContext" class="headerlink" title="SourceContext"></a>SourceContext<t></t></h3><ul>
<li><p>SourceContext是SourceFunction的内部接口，用于发射数据和watermarks</p>
</li>
<li><p>一共包含六个方法：</p>
<blockquote>
<p>1、collect：发射数据，没有timestamp</p>
<p>2、collectWithTimestamp</p>
<p>3、emitWatermark</p>
<p>4、markAsTemporarilyIdle</p>
<p>5、getCheckpointLock：应该是类级别锁</p>
<p>6、close：系统关闭context</p>
</blockquote>
</li>
</ul>
<h1 id="RichFunction"><a href="#RichFunction" class="headerlink" title="RichFunction"></a>RichFunction</h1><ul>
<li><p>所有rich UDF的基类</p>
</li>
<li><p>该类定义了函数的生命周期，并且可以获取运行函数的context</p>
</li>
<li><p>该接口包含5个方法</p>
<blockquote>
<p>1、open：函数的初始化方法，该方法可以使用任务的参数对象</p>
<p>2、close：用于清理工作</p>
<p>3、getRuntimeContext：类的运行时信息</p>
<p>4、getIterationRuntimeContext：获取特定版本的RuntimeContext，</p>
<p>5、setRuntimeContext：框架用来设置RuntimeContext</p>
</blockquote>
</li>
</ul>
<h1 id="算子"><a href="#算子" class="headerlink" title="算子"></a>算子</h1><h2 id="StreamOperator"><a href="#StreamOperator" class="headerlink" title="StreamOperator"></a>StreamOperator</h2><ul>
<li><p>stream算子的基接口</p>
</li>
<li><p>所有的方法保证不被并发调用</p>
</li>
<li><p>该接口共包含9个方法</p>
<blockquote>
<p>1、open：用于算子的初始化逻辑，在任意元素被处理之前调用</p>
<p>2、close：当所有元素被添加到算子后调用，用于flush buffer</p>
<p>3、dispose：在算子的生命周期的最后调用，用于释放资源</p>
<p>4、prepareSnapshotPreBarrier：当算子应该做snapshot，在发射自己的checkpoint barrier之前调用</p>
<p>​    不做实际的状态持久化，应用于一些小的事务处理</p>
<p>5、snapshotState：做该算子的状态snapshot</p>
<p>6、initializeState：</p>
<p>7、setKeyContextElement1  setKeyContextElement2 </p>
<p>8、getMetricGroup</p>
<p>9、getOperatorID</p>
</blockquote>
</li>
</ul>
<h2 id="SetupableStreamOperator"><a href="#SetupableStreamOperator" class="headerlink" title="SetupableStreamOperator"></a>SetupableStreamOperator</h2><ul>
<li><p>用来实现访问 context and the output</p>
</li>
<li><p>共包含3个方法：</p>
<blockquote>
<p>1、setup</p>
<p>2、getChainingStrategy</p>
<p>3、setChainingStrategy</p>
</blockquote>
</li>
</ul>
<h2 id="CheckpointedStreamOperator"><a href="#CheckpointedStreamOperator" class="headerlink" title="CheckpointedStreamOperator"></a>CheckpointedStreamOperator</h2><ul>
<li><p>自定义的状态处理hook，其被StreamOperatorStateHandler调用</p>
</li>
<li><p>包含两个方法</p>
<blockquote>
<p>1、initializeState</p>
<p>2、snapshotState</p>
</blockquote>
</li>
</ul>
<h2 id="AbstractStreamOperator"><a href="#AbstractStreamOperator" class="headerlink" title="AbstractStreamOperator"></a>AbstractStreamOperator</h2><ul>
<li>所有stream operators的基类</li>
<li></li>
</ul>
<h1 id="源数据格式"><a href="#源数据格式" class="headerlink" title="源数据格式"></a>源数据格式</h1><h2 id="InputSplit"><a href="#InputSplit" class="headerlink" title="InputSplit"></a>InputSplit</h2><ul>
<li><p>InputSplit定义了由InputFormat产生的数据部分</p>
</li>
<li><p>该接口由被分配给input formats 的  input splits来实现</p>
</li>
<li><p>该接口包含一个方法</p>
<blockquote>
<p>1、getSplitNumber：返回此输入拆分的编号。</p>
</blockquote>
</li>
</ul>
<h2 id="InputSplitSource"><a href="#InputSplitSource" class="headerlink" title="InputSplitSource"></a>InputSplitSource</h2><ul>
<li><p>用于创建InputSplit</p>
</li>
<li><p>包含两个方法</p>
<blockquote>
<p>1、T[] createInputSplits(int minNumSplits)</p>
<p>2、InputSplitAssigner getInputSplitAssigner(T[] inputSplits)：Assigner决定了input format的并行度实例读取哪个input split</p>
</blockquote>
</li>
</ul>
<h2 id="InputSplitAssigner"><a href="#InputSplitAssigner" class="headerlink" title="InputSplitAssigner"></a>InputSplitAssigner</h2><h2 id="InputFormat"><a href="#InputFormat" class="headerlink" title="InputFormat"></a>InputFormat</h2><ul>
<li><p>产生记录的数据源的基本接口</p>
</li>
<li><p>基本功能如下</p>
<blockquote>
<p>1、它描述了如何将输入 拆分为可以并行处理的拆分</p>
<p>2、它描述了如何从输入拆分中读取记录</p>
<p>3、它描述了如何从输入中收集基本统计信息</p>
</blockquote>
</li>
</ul>

        
    </div>

    
      <footer class="post-footer">
        <div class="post-eof"></div>
      </footer>
  </article>
  
  
  <article itemscope itemtype="http://schema.org/Article" class="post-block home" lang="zh-CN">
    <link itemprop="mainEntityOfPage" href="http://yoursite.com/2021/04/21/java/秒杀系统设计/">

    <span hidden itemprop="author" itemscope itemtype="http://schema.org/Person">
      <meta itemprop="name" content="姜纪光">
      <meta itemprop="description" content="">
      <meta itemprop="image" content="/images/avatar.gif">
    </span>

    <span hidden itemprop="publisher" itemscope itemtype="http://schema.org/Organization">
      <meta itemprop="name" content="纪光的博客">
    </span>
      <header class="post-header">
        <h1 class="post-title" itemprop="name headline">
          
            
            <a href="/2021/04/21/java/秒杀系统设计/" class="post-title-link" itemprop="url">秒杀系统设计</a>
          
        </h1>

        <div class="post-meta">
            <span class="post-meta-item">
              <span class="post-meta-item-icon">
                <i class="fa fa-calendar-o"></i>
              </span>
              <span class="post-meta-item-text">发表于</span>

              
              <time title="创建时间：2021-04-21 08:48:43" itemprop="dateCreated datePublished" datetime="2021-04-21T08:48:43+08:00">2021-04-21</time>
            </span>
          
            
              <span class="post-meta-item">
                <span class="post-meta-item-icon">
                  <i class="fa fa-calendar-check-o"></i>
                </span>
                <span class="post-meta-item-text">更新于</span>
                <time title="修改时间：2021-06-20 12:19:35" itemprop="dateModified" datetime="2021-06-20T12:19:35+08:00">2021-06-20</time>
              </span>
            
          
            <span class="post-meta-item">
              <span class="post-meta-item-icon">
                <i class="fa fa-folder-o"></i>
              </span>
              <span class="post-meta-item-text">分类于</span>
              
                <span itemprop="about" itemscope itemtype="http://schema.org/Thing">
                  <a href="/categories/java/" itemprop="url" rel="index">
                    <span itemprop="name">java</span>
                  </a>
                </span>

                
            </span>
          

            <span id="/2021/04/21/java/秒杀系统设计/" class="post-meta-item leancloud_visitors" data-flag-title="秒杀系统设计" title="阅读次数">
              <span class="post-meta-item-icon">
                <i class="fa fa-eye"></i>
              </span>
              <span class="post-meta-item-text">阅读次数：</span>
              <span class="leancloud-visitors-count"></span>
            </span>
          
  
        </div>
      </header>

    
    <div class="post-body" itemprop="articleBody">

      
          <h1 id="系统目标"><a href="#系统目标" class="headerlink" title="系统目标"></a>系统目标</h1><ul>
<li>高性能</li>
<li>高可用</li>
<li>一致性</li>
</ul>
<h1 id="面临的问题"><a href="#面临的问题" class="headerlink" title="面临的问题"></a>面临的问题</h1><ul>
<li>同一时刻有超大量的并发读和并发写</li>
<li></li>
</ul>
<h1 id="设计方案"><a href="#设计方案" class="headerlink" title="设计方案"></a>设计方案</h1><h2 id="架构原则"><a href="#架构原则" class="headerlink" title="架构原则"></a>架构原则</h2><ul>
<li><p>最终的核心方案就是一个字：减</p>
</li>
<li><p>数据要尽量少</p>
</li>
<li><p>请求数要尽量少</p>
</li>
<li><p>路径要尽量短：all in one</p>
</li>
<li><p>依赖要尽量少</p>
</li>
<li><p>不要有单点</p>
</li>
<li><p>把秒杀单独出来作为一个独立系统</p>
</li>
<li><p>热点数据缓存</p>
</li>
<li><p>秒杀答题，防止秒杀机器抢单</p>
</li>
<li><p>商品种类错峰秒杀</p>
</li>
<li><p>一段时间内禁止请求</p>
</li>
</ul>
<h2 id="高性能"><a href="#高性能" class="headerlink" title="高性能"></a>高性能</h2><ul>
<li><p>数据的动静分离方案</p>
<blockquote>
<p>1、把静态数据缓存到离用户最近的地方：浏览器、网关层、服务端cache中</p>
<p>2、缓存http链接</p>
<p>3、Cache 做 Hash 分组，一组 Cache 缓存的内容相同，避免热点数据过度集中导致新的瓶颈产生</p>
</blockquote>
</li>
<li><p>热点的发现与隔离</p>
<blockquote>
<p>1、异步抓取热点数据</p>
<p>2、热点数据发现事实</p>
<p>3、处理热点数据：优化（缓存）、限制（根据hash做分桶、每个分桶设置一个处理队列）、隔离（业务、系统、数据隔离）</p>
</blockquote>
</li>
<li><p>请求的削峰与分层过滤</p>
<blockquote>
<p>1、排队，使用消息队列缓冲瞬时流量</p>
<p>2、答题</p>
<p>3、分层过滤</p>
</blockquote>
</li>
<li><p>服务端的极致优化</p>
</li>
</ul>
<h2 id="一致性"><a href="#一致性" class="headerlink" title="一致性"></a>一致性</h2><ul>
<li>秒杀减库存方案</li>
</ul>
<h2 id="高可用"><a href="#高可用" class="headerlink" title="高可用"></a>高可用</h2><ul>
<li>减少功能</li>
<li>独立优化</li>
<li></li>
</ul>
<h2 id="“减库存”设计的核心逻辑"><a href="#“减库存”设计的核心逻辑" class="headerlink" title="“减库存”设计的核心逻辑"></a>“减库存”设计的核心逻辑</h2><ul>
<li>下单减库存</li>
<li>预扣库存</li>
</ul>
<h2 id="兜底方案"><a href="#兜底方案" class="headerlink" title="兜底方案"></a>兜底方案</h2><ul>
<li>降级</li>
<li>限流</li>
<li>拒绝服务</li>
</ul>
<h1 id="设计方案2"><a href="#设计方案2" class="headerlink" title="设计方案2"></a>设计方案2</h1><h2 id="后端优化"><a href="#后端优化" class="headerlink" title="后端优化"></a>后端优化</h2><ul>
<li>将请求尽量拦截在系统上游</li>
<li>限流：屏蔽掉无用的流量</li>
<li>削峰：缓冲瞬时流量</li>
<li>异步：同步请求转换为异步异步请求</li>
<li>缓存：本地、redis</li>
<li>负载均衡</li>
</ul>
<h2 id="前端优化"><a href="#前端优化" class="headerlink" title="前端优化"></a>前端优化</h2><ul>
<li>限流：答题</li>
<li>禁止重复提交</li>
<li>动静分离</li>
</ul>
<h2 id="防作弊优化"><a href="#防作弊优化" class="headerlink" title="防作弊优化"></a>防作弊优化</h2><ul>
<li>防爬虫</li>
<li></li>
</ul>

        
    </div>

    
      <footer class="post-footer">
        <div class="post-eof"></div>
      </footer>
  </article>
  
  
  <article itemscope itemtype="http://schema.org/Article" class="post-block home" lang="zh-CN">
    <link itemprop="mainEntityOfPage" href="http://yoursite.com/2021/04/15/spark/Spark3-0新特性/">

    <span hidden itemprop="author" itemscope itemtype="http://schema.org/Person">
      <meta itemprop="name" content="姜纪光">
      <meta itemprop="description" content="">
      <meta itemprop="image" content="/images/avatar.gif">
    </span>

    <span hidden itemprop="publisher" itemscope itemtype="http://schema.org/Organization">
      <meta itemprop="name" content="纪光的博客">
    </span>
      <header class="post-header">
        <h1 class="post-title" itemprop="name headline">
          
            
            <a href="/2021/04/15/spark/Spark3-0新特性/" class="post-title-link" itemprop="url">Spark3.0新特性</a>
          
        </h1>

        <div class="post-meta">
            <span class="post-meta-item">
              <span class="post-meta-item-icon">
                <i class="fa fa-calendar-o"></i>
              </span>
              <span class="post-meta-item-text">发表于</span>

              
              <time title="创建时间：2021-04-15 16:49:24" itemprop="dateCreated datePublished" datetime="2021-04-15T16:49:24+08:00">2021-04-15</time>
            </span>
          
            
              <span class="post-meta-item">
                <span class="post-meta-item-icon">
                  <i class="fa fa-calendar-check-o"></i>
                </span>
                <span class="post-meta-item-text">更新于</span>
                <time title="修改时间：2021-06-20 12:19:35" itemprop="dateModified" datetime="2021-06-20T12:19:35+08:00">2021-06-20</time>
              </span>
            
          
            <span class="post-meta-item">
              <span class="post-meta-item-icon">
                <i class="fa fa-folder-o"></i>
              </span>
              <span class="post-meta-item-text">分类于</span>
              
                <span itemprop="about" itemscope itemtype="http://schema.org/Thing">
                  <a href="/categories/spark/" itemprop="url" rel="index">
                    <span itemprop="name">spark</span>
                  </a>
                </span>

                
            </span>
          

            <span id="/2021/04/15/spark/Spark3-0新特性/" class="post-meta-item leancloud_visitors" data-flag-title="Spark3.0新特性" title="阅读次数">
              <span class="post-meta-item-icon">
                <i class="fa fa-eye"></i>
              </span>
              <span class="post-meta-item-text">阅读次数：</span>
              <span class="leancloud-visitors-count"></span>
            </span>
          
  
        </div>
      </header>

    
    <div class="post-body" itemprop="articleBody">

      
          <h1 id="自适应执行"><a href="#自适应执行" class="headerlink" title="自适应执行"></a>自适应执行</h1><ul>
<li><p>思路</p>
<blockquote>
<p>首先要知道这个等式a join b = a1 join b union a2 join b,</p>
<p>AE 通过把大的分区段分别join然后union就可以改善数据倾斜的问题</p>
</blockquote>
</li>
<li><p>引入QueryStage,将一个大的执行计划切成多个子执行计划，对Job实现执行中动态优化</p>
</li>
<li><p>依赖上个QueryStage执行产生的shuffle数据，在执行当前QueryStage的时候优化并行度</p>
</li>
</ul>
<h1 id="动态分区裁剪"><a href="#动态分区裁剪" class="headerlink" title="动态分区裁剪"></a>动态分区裁剪</h1>
        
      
    </div>

    
      <footer class="post-footer">
        <div class="post-eof"></div>
      </footer>
  </article>
  
  
  <article itemscope itemtype="http://schema.org/Article" class="post-block home" lang="zh-CN">
    <link itemprop="mainEntityOfPage" href="http://yoursite.com/2021/04/15/spark/Spark内存模型/">

    <span hidden itemprop="author" itemscope itemtype="http://schema.org/Person">
      <meta itemprop="name" content="姜纪光">
      <meta itemprop="description" content="">
      <meta itemprop="image" content="/images/avatar.gif">
    </span>

    <span hidden itemprop="publisher" itemscope itemtype="http://schema.org/Organization">
      <meta itemprop="name" content="纪光的博客">
    </span>
      <header class="post-header">
        <h1 class="post-title" itemprop="name headline">
          
            
            <a href="/2021/04/15/spark/Spark内存模型/" class="post-title-link" itemprop="url">Spark内存模型</a>
          
        </h1>

        <div class="post-meta">
            <span class="post-meta-item">
              <span class="post-meta-item-icon">
                <i class="fa fa-calendar-o"></i>
              </span>
              <span class="post-meta-item-text">发表于</span>

              
              <time title="创建时间：2021-04-15 15:16:32" itemprop="dateCreated datePublished" datetime="2021-04-15T15:16:32+08:00">2021-04-15</time>
            </span>
          
            
              <span class="post-meta-item">
                <span class="post-meta-item-icon">
                  <i class="fa fa-calendar-check-o"></i>
                </span>
                <span class="post-meta-item-text">更新于</span>
                <time title="修改时间：2021-06-20 12:19:35" itemprop="dateModified" datetime="2021-06-20T12:19:35+08:00">2021-06-20</time>
              </span>
            
          
            <span class="post-meta-item">
              <span class="post-meta-item-icon">
                <i class="fa fa-folder-o"></i>
              </span>
              <span class="post-meta-item-text">分类于</span>
              
                <span itemprop="about" itemscope itemtype="http://schema.org/Thing">
                  <a href="/categories/spark/" itemprop="url" rel="index">
                    <span itemprop="name">spark</span>
                  </a>
                </span>

                
            </span>
          

            <span id="/2021/04/15/spark/Spark内存模型/" class="post-meta-item leancloud_visitors" data-flag-title="Spark内存模型" title="阅读次数">
              <span class="post-meta-item-icon">
                <i class="fa fa-eye"></i>
              </span>
              <span class="post-meta-item-text">阅读次数：</span>
              <span class="leancloud-visitors-count"></span>
            </span>
          
  
        </div>
      </header>

    
    <div class="post-body" itemprop="articleBody">

      
          <h1 id="Executor内存"><a href="#Executor内存" class="headerlink" title="Executor内存"></a>Executor内存</h1><ul>
<li>Driver的内存管理比较简单，和一般的JVM程序区别不大，所以本文重点分析Executor的内存管理</li>
<li>大致分为堆内和堆外内存</li>
</ul>
<h1 id="堆内内存"><a href="#堆内内存" class="headerlink" title="堆内内存"></a>堆内内存</h1><ul>
<li><p>通过spark.executor.memory参数设置</p>
</li>
<li><p>Executor内运行的并发任务共享JVM堆内内存</p>
</li>
<li><p>堆内内存分为4个部分</p>
<blockquote>
<p>1、<strong>Storage内存</strong>，主要用于存储Spark的cache数据，例如RDD的cache，Broadcast变量</p>
<p>​    默认为统一内存的0.5</p>
<p>2、<strong>Execution内存</strong>，用于存储task执行过程中需要的对象，如Shuffle、Join、Sort、Aggregation等计算过程中的临时数据</p>
<p>  为统一内存 - <strong>Storage内存</strong></p>
<p>3、<strong>User内存</strong>，用来执行用户的代码</p>
<p>4、<strong>Reserved内存</strong>：这部分内存是预留给系统使用，是固定不变的</p>
</blockquote>
</li>
</ul>
<h1 id="堆外内存"><a href="#堆外内存" class="headerlink" title="堆外内存"></a>堆外内存</h1><ul>
<li><p>从概念上分为两个：</p>
<blockquote>
<p>Executor JVM内的堆外内存，主要用于JVM自身，字符串, NIO Buffer等开销</p>
<p>直接在work节点上的堆外内存，提高Shuffle时排序效率</p>
</blockquote>
</li>
</ul>
<h1 id="统一内存管理"><a href="#统一内存管理" class="headerlink" title="统一内存管理"></a>统一内存管理</h1>
        
      
    </div>

    
      <footer class="post-footer">
        <div class="post-eof"></div>
      </footer>
  </article>
  
  
  <article itemscope itemtype="http://schema.org/Article" class="post-block home" lang="zh-CN">
    <link itemprop="mainEntityOfPage" href="http://yoursite.com/2021/04/14/spark/SparkSQL调优/">

    <span hidden itemprop="author" itemscope itemtype="http://schema.org/Person">
      <meta itemprop="name" content="姜纪光">
      <meta itemprop="description" content="">
      <meta itemprop="image" content="/images/avatar.gif">
    </span>

    <span hidden itemprop="publisher" itemscope itemtype="http://schema.org/Organization">
      <meta itemprop="name" content="纪光的博客">
    </span>
      <header class="post-header">
        <h1 class="post-title" itemprop="name headline">
          
            
            <a href="/2021/04/14/spark/SparkSQL调优/" class="post-title-link" itemprop="url">SparkSQL调优</a>
          
        </h1>

        <div class="post-meta">
            <span class="post-meta-item">
              <span class="post-meta-item-icon">
                <i class="fa fa-calendar-o"></i>
              </span>
              <span class="post-meta-item-text">发表于</span>

              
              <time title="创建时间：2021-04-14 17:14:25" itemprop="dateCreated datePublished" datetime="2021-04-14T17:14:25+08:00">2021-04-14</time>
            </span>
          
            
              <span class="post-meta-item">
                <span class="post-meta-item-icon">
                  <i class="fa fa-calendar-check-o"></i>
                </span>
                <span class="post-meta-item-text">更新于</span>
                <time title="修改时间：2021-06-20 12:19:35" itemprop="dateModified" datetime="2021-06-20T12:19:35+08:00">2021-06-20</time>
              </span>
            
          
            <span class="post-meta-item">
              <span class="post-meta-item-icon">
                <i class="fa fa-folder-o"></i>
              </span>
              <span class="post-meta-item-text">分类于</span>
              
                <span itemprop="about" itemscope itemtype="http://schema.org/Thing">
                  <a href="/categories/spark/" itemprop="url" rel="index">
                    <span itemprop="name">spark</span>
                  </a>
                </span>

                
            </span>
          

            <span id="/2021/04/14/spark/SparkSQL调优/" class="post-meta-item leancloud_visitors" data-flag-title="SparkSQL调优" title="阅读次数">
              <span class="post-meta-item-icon">
                <i class="fa fa-eye"></i>
              </span>
              <span class="post-meta-item-text">阅读次数：</span>
              <span class="leancloud-visitors-count"></span>
            </span>
          
  
        </div>
      </header>

    
    <div class="post-body" itemprop="articleBody">

      
          <h1 id="简介"><a href="#简介" class="headerlink" title="简介"></a>简介</h1><ul>
<li><p>分布式计算引擎在调优方面有四个主要关注方向，分别是CPU、内存、网络开销和I/O，其具体调优目标如下：</p>
<blockquote>
<p>1、提高CPU利用率</p>
<p>2、避免OOM</p>
<p>3、降低网络开销</p>
<p>4、减少I/O操作</p>
</blockquote>
</li>
</ul>
<h1 id="调优参数"><a href="#调优参数" class="headerlink" title="调优参数"></a>调优参数</h1><h2 id="executor"><a href="#executor" class="headerlink" title="executor"></a>executor</h2><ul>
<li><p>spark.executor.cores</p>
<blockquote>
<p>作用：单个executor内任务的并发数，这些任务共享executor内存，因此不宜设置过大，容易造成oom</p>
<p>默认值1，建议值2</p>
</blockquote>
</li>
<li><p>spark.executor.memory</p>
<blockquote>
<p>作用：各个 executor 使用的堆内内存，不可超过单机的最大可使用内存。</p>
<p>默认值2g，建议值12g</p>
</blockquote>
</li>
<li><p>spark.yarn.executor.memoryOverhead</p>
<blockquote>
<p>单位MB，默认值3072</p>
<p>作用：包括JVM overhead，sort、shuffle以及Netty的堆外内存等。</p>
<p>限制： spark.executor.memory + spark.yarn.executor.memoryOverhead &lt;= 16G</p>
</blockquote>
</li>
</ul>
<h2 id="driver"><a href="#driver" class="headerlink" title="driver"></a>driver</h2><ul>
<li><p>spark.driver.memory</p>
<blockquote>
<p>driver的堆内内存，默认2g，建议12g</p>
</blockquote>
</li>
<li><p>spark.yarn.driver.memoryOverhead</p>
<blockquote>
<p>driver的堆外内存，默认3072，建议3072</p>
</blockquote>
</li>
</ul>
<h2 id="动态分区"><a href="#动态分区" class="headerlink" title="动态分区"></a>动态分区</h2><ul>
<li><p>hive.exec.dynamic.partition</p>
<blockquote>
<p>支持动态分区，默认值false，建议值true</p>
</blockquote>
</li>
<li><p>hive.exec.dynamic.partition.mode</p>
<blockquote>
<p>当有一个分区是静态分区时：strict 其他设置为：nonstrict</p>
</blockquote>
</li>
<li><p>hive.exec.max.dynamic.partitions</p>
<blockquote>
<p>允许的分区总数，建议值100000</p>
</blockquote>
</li>
<li><p>hive.exec.max.dynamic.partitions.pernode</p>
<blockquote>
<p>每个执行节点允许的最大分区数，应该是executor节点</p>
<p>默认值100，建议值100000</p>
</blockquote>
</li>
<li><p>hive.exec.max.created.files</p>
<blockquote>
<p>所有map和reduce任务可以产生的hdfs文件的和</p>
<p>默认值100000，建议值150000</p>
</blockquote>
</li>
<li><p>hive.error.on.empty.partition</p>
<blockquote>
<p>当有空分区生成时，是否抛出异常，默认值false</p>
</blockquote>
</li>
</ul>
<h2 id="join"><a href="#join" class="headerlink" title="join"></a>join</h2><ul>
<li><p>spark.sql.autoBroadcastJoinThreshold</p>
<blockquote>
<p>1、当大表JOIN小表时，如果小表足够小，可以将大表分片，分别用小表和每个大表的分片进行JOIN，最后汇总，能够大大提升作业性能。</p>
<p>2、如果小表的大小小于该值，则会将小表广播到所有executor中，使JOIN快速完成。如果该值设置太大，则会导致executor内存压力过大，容易出现OOM。</p>
<p>默认值25M，建议值100M</p>
</blockquote>
</li>
<li><p>hive.auto.convert.join</p>
<blockquote>
<p>自动将 Reduce 端的 Common Join 转化为 Map Join，从而加快大表关联小表的 Join 速度</p>
</blockquote>
</li>
</ul>
<h2 id="orc"><a href="#orc" class="headerlink" title="orc"></a>orc</h2><ul>
<li><p>spark.sql.orc.filterPushdown</p>
<blockquote>
<p>当为true时，开启filter的下推功能</p>
<p>默认值false，建议值true</p>
</blockquote>
</li>
<li><p>hive.exec.orc.split.strategy</p>
<blockquote>
<p>ETL：会切分文件,多个stripe组成一个split</p>
<p>BI：按文件进行切分</p>
<p>HYBRID：平均文件大小大于hadoop最大split值使用 ETL,否则BI</p>
</blockquote>
</li>
<li><p>spark.sql.orc.splits.include.file.footer</p>
<blockquote>
<p>官网建议true</p>
</blockquote>
</li>
<li><p>spark.sql.orc.cache.stripe.details.size</p>
<blockquote>
<p>建议值10000</p>
</blockquote>
</li>
<li><p>spark.sql.hive.metastorePartitionPruning</p>
<blockquote>
<p>当为true时，一些谓词将被下推到hive的metastore，以便于不匹配的分区能够及早清除。仅仅作用于hive表不能被转换成ilesource relations</p>
</blockquote>
</li>
<li><p>spark.hadoop.mapreduce.input.fileinputformat.split.maxsize</p>
<blockquote>
<p>读ORC时，设置一个split的最大值，超出后会进行文件切分</p>
<p>建议值128M</p>
</blockquote>
</li>
<li><p>spark.hadoop.mapreduce.input.fileinputformat.split.minsize</p>
<blockquote>
<p>读ORC时，设置小文件合并的阈值</p>
<p>建议值64M</p>
</blockquote>
</li>
</ul>
<h2 id="内存管理"><a href="#内存管理" class="headerlink" title="内存管理"></a>内存管理</h2><ul>
<li><p>spark.memory.fraction</p>
<blockquote>
<p>统一内存（storage+execution）所占比例      统一内存=（jvm内存-预留内存 ）* 统一内存所占比例</p>
<p>保留内存，防止产生oom，默认为300M</p>
<p>其它内存，用于执行用户的代码的内存        其它内存 = （jvm内存-预留内存 ）- 统一内存</p>
<p>建议值0.6</p>
</blockquote>
</li>
<li><p>spark.storage.storageFraction</p>
<blockquote>
<p>storage内存所占比例，用于缓存数据   storage内存= 统一内存 * storage内存所占比例</p>
<p>execution执行内存，用于缓存在执行shuffle过程中产生的中间数据和直接输出的部分缓存数据</p>
<p>execution内存=统一内存 - storage内存</p>
<p>执行内存的空间被对方占用后，可让对方将占用的部分转存到硬盘，然后”归还”借用的空间</p>
<p>存储内存的空间被对方占用后，无法让对方”归还”</p>
</blockquote>
</li>
<li><p>数据缓存方式</p>
<blockquote>
<p>MEMORY_ONLY 仅仅放在内存中，如果内存放不下就要清除</p>
<p>MEMORY_AND_DISK 优先内存，内存放不下就写到磁盘</p>
<p>MEMORY_ONLY_SER 以序列化方式存放到内存</p>
<p>MEMORY_AND_DISK_SER 以序列化方式存放到内存和磁盘</p>
<p>垃圾回收的代价和堆里的对象数据相关，而不是和数据的字节数相关。</p>
<p>序列化的方式会把大量对象序列化为一个巨大的缓存区对象，如果缓存的数据量比较大或者有gc时间过长</p>
<p>可以使用序列化的方式。</p>
</blockquote>
</li>
</ul>
<h2 id="动态资源分配"><a href="#动态资源分配" class="headerlink" title="动态资源分配"></a>动态资源分配</h2><ul>
<li><p>spark.dynamicAllocation.enabled</p>
<blockquote>
<p>开启动态资源分配，默认值true，建议值true</p>
</blockquote>
</li>
<li><p>spark.dynamicAllocation.maxExecutors</p>
<blockquote>
<p>每个application可以同时拥有的最大Executor数量，建议值1000</p>
</blockquote>
</li>
<li><p>spark.dynamicAllocation.minExecutors</p>
<blockquote>
<p>每个application可以同时拥有的最小Executor数量，建议值3</p>
</blockquote>
</li>
</ul>
<h2 id="shuffle"><a href="#shuffle" class="headerlink" title="shuffle"></a>shuffle</h2><ul>
<li><p>spark.sql.shuffle.partitions</p>
<blockquote>
<p>该参数代表了shuffle read task的并行度</p>
<p>该参数应该和spark.dynamicAllocation.maxExecutors、spark.executor.cores配合使用应该为上面两个参数的整数倍</p>
</blockquote>
</li>
<li></li>
<li><p>spark.shuffle.file.buffer</p>
<blockquote>
<p>该参数用于设置shuffle write task的BufferedOutputStream的buffer缓冲大小。将数据写到磁盘文件之前，会先写入buffer缓冲中，待缓冲写满之后，才会溢写到磁盘。</p>
<p>默认值32k  建议值64k</p>
</blockquote>
</li>
<li><p>spark.reducer.maxSizeInFlight</p>
<blockquote>
<p>该参数用于设置shuffle read task的buffer缓冲大小，而这个buffer缓冲决定了每次能够拉取多少数据。</p>
<p>改值太大容易导致reduce oom</p>
<p>默认值48m  建议值96m</p>
</blockquote>
</li>
</ul>

        
    </div>

    
      <footer class="post-footer">
        <div class="post-eof"></div>
      </footer>
  </article>
  
  
  <article itemscope itemtype="http://schema.org/Article" class="post-block home" lang="zh-CN">
    <link itemprop="mainEntityOfPage" href="http://yoursite.com/2021/04/14/spark/Spark基本知识/">

    <span hidden itemprop="author" itemscope itemtype="http://schema.org/Person">
      <meta itemprop="name" content="姜纪光">
      <meta itemprop="description" content="">
      <meta itemprop="image" content="/images/avatar.gif">
    </span>

    <span hidden itemprop="publisher" itemscope itemtype="http://schema.org/Organization">
      <meta itemprop="name" content="纪光的博客">
    </span>
      <header class="post-header">
        <h1 class="post-title" itemprop="name headline">
          
            
            <a href="/2021/04/14/spark/Spark基本知识/" class="post-title-link" itemprop="url">Spark基本知识</a>
          
        </h1>

        <div class="post-meta">
            <span class="post-meta-item">
              <span class="post-meta-item-icon">
                <i class="fa fa-calendar-o"></i>
              </span>
              <span class="post-meta-item-text">发表于</span>

              
              <time title="创建时间：2021-04-14 17:04:57" itemprop="dateCreated datePublished" datetime="2021-04-14T17:04:57+08:00">2021-04-14</time>
            </span>
          
            
              <span class="post-meta-item">
                <span class="post-meta-item-icon">
                  <i class="fa fa-calendar-check-o"></i>
                </span>
                <span class="post-meta-item-text">更新于</span>
                <time title="修改时间：2021-06-20 12:19:35" itemprop="dateModified" datetime="2021-06-20T12:19:35+08:00">2021-06-20</time>
              </span>
            
          
            <span class="post-meta-item">
              <span class="post-meta-item-icon">
                <i class="fa fa-folder-o"></i>
              </span>
              <span class="post-meta-item-text">分类于</span>
              
                <span itemprop="about" itemscope itemtype="http://schema.org/Thing">
                  <a href="/categories/spark/" itemprop="url" rel="index">
                    <span itemprop="name">spark</span>
                  </a>
                </span>

                
            </span>
          

            <span id="/2021/04/14/spark/Spark基本知识/" class="post-meta-item leancloud_visitors" data-flag-title="Spark基本知识" title="阅读次数">
              <span class="post-meta-item-icon">
                <i class="fa fa-eye"></i>
              </span>
              <span class="post-meta-item-text">阅读次数：</span>
              <span class="leancloud-visitors-count"></span>
            </span>
          
  
        </div>
      </header>

    
    <div class="post-body" itemprop="articleBody">

      
          <h1 id="基本概念"><a href="#基本概念" class="headerlink" title="基本概念"></a>基本概念</h1><ul>
<li><p>RDD</p>
<blockquote>
<p>只读、分布式、弹性、基于内存</p>
</blockquote>
</li>
<li><p>有向无环DAG</p>
<blockquote>
<p>按照宽依赖进行stage划分</p>
</blockquote>
</li>
<li><p>Application：表示你的应用程序</p>
</li>
<li><p>Driver</p>
<blockquote>
<p>1、创建Spark上下文</p>
<p>2、提交Spark作业（Job）并将Job转化为计算任务（Task）交给Executor计算</p>
<p>3、协调各个Executor进程间任务调度</p>
</blockquote>
</li>
<li><p>Executor</p>
<blockquote>
<p>1、在工作节点上执行具体的计算任务（Task），并角计算结果返回给Driver</p>
<p>2、为需要持久化的RDD提供存储功能</p>
</blockquote>
</li>
<li><p>Worker：集群中可以运行Application代码的节点。在Standalone模式中指的是通过slave文件配置的worker节点，在Spark on Yarn模式中指的就是NodeManager节点。</p>
</li>
<li><p>Task：在Executor进程中执行任务的工作单元，多个Task组成一个Stage</p>
</li>
<li><p>Job：包含多个Task组成的并行计算，是由Action行为触发的。也就是每个action会触发一个job</p>
</li>
<li><p>Stage：每个Job会被拆分很多组Task，作为一个TaskSet，其名称为Stage</p>
</li>
<li><p>DAGScheduler：根据Job构建基于Stage的DAG，并提交Stage给TaskScheduler，其划分Stage的依据是RDD之间的依赖关系</p>
</li>
<li><p>TaskScheduler：将TaskSet提交给Worker（集群）运行，每个Executor运行什么Task就是在此处分配的。</p>
</li>
<li><p>Application Master</p>
<blockquote>
<p>在YARN中，每个Application实例都有一个ApplicationMaster进程，它是Application启动的第一个容器。它负责和ResourceManager打交道并请求资源，获取资源之后告诉NodeManager为其启动Container。从深层次的含义讲YARN-Cluster和YARN-Client模式的区别其实就是ApplicationMaster进程的区别。</p>
</blockquote>
</li>
</ul>

        
    </div>

    
      <footer class="post-footer">
        <div class="post-eof"></div>
      </footer>
  </article>
  
  
  <article itemscope itemtype="http://schema.org/Article" class="post-block home" lang="zh-CN">
    <link itemprop="mainEntityOfPage" href="http://yoursite.com/2021/04/07/数据湖/Iceberg源码贡献/">

    <span hidden itemprop="author" itemscope itemtype="http://schema.org/Person">
      <meta itemprop="name" content="姜纪光">
      <meta itemprop="description" content="">
      <meta itemprop="image" content="/images/avatar.gif">
    </span>

    <span hidden itemprop="publisher" itemscope itemtype="http://schema.org/Organization">
      <meta itemprop="name" content="纪光的博客">
    </span>
      <header class="post-header">
        <h1 class="post-title" itemprop="name headline">
          
            
            <a href="/2021/04/07/数据湖/Iceberg源码贡献/" class="post-title-link" itemprop="url">Iceberg源码贡献</a>
          
        </h1>

        <div class="post-meta">
            <span class="post-meta-item">
              <span class="post-meta-item-icon">
                <i class="fa fa-calendar-o"></i>
              </span>
              <span class="post-meta-item-text">发表于</span>

              
              <time title="创建时间：2021-04-07 14:59:05" itemprop="dateCreated datePublished" datetime="2021-04-07T14:59:05+08:00">2021-04-07</time>
            </span>
          
            
              <span class="post-meta-item">
                <span class="post-meta-item-icon">
                  <i class="fa fa-calendar-check-o"></i>
                </span>
                <span class="post-meta-item-text">更新于</span>
                <time title="修改时间：2021-06-20 12:19:35" itemprop="dateModified" datetime="2021-06-20T12:19:35+08:00">2021-06-20</time>
              </span>
            
          
            <span class="post-meta-item">
              <span class="post-meta-item-icon">
                <i class="fa fa-folder-o"></i>
              </span>
              <span class="post-meta-item-text">分类于</span>
              
                <span itemprop="about" itemscope itemtype="http://schema.org/Thing">
                  <a href="/categories/iceberg/" itemprop="url" rel="index">
                    <span itemprop="name">iceberg</span>
                  </a>
                </span>

                
            </span>
          

            <span id="/2021/04/07/数据湖/Iceberg源码贡献/" class="post-meta-item leancloud_visitors" data-flag-title="Iceberg源码贡献" title="阅读次数">
              <span class="post-meta-item-icon">
                <i class="fa fa-eye"></i>
              </span>
              <span class="post-meta-item-text">阅读次数：</span>
              <span class="leancloud-visitors-count"></span>
            </span>
          
  
        </div>
      </header>

    
    <div class="post-body" itemprop="articleBody">

      
          <h1 id="源码贡献"><a href="#源码贡献" class="headerlink" title="源码贡献"></a>源码贡献</h1><h2 id="代码仓库"><a href="#代码仓库" class="headerlink" title="代码仓库"></a>代码仓库</h2><figure class="highlight shell"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br></pre></td><td class="code"><pre><span class="line"><span class="meta">#</span> clone自己的仓库</span><br><span class="line"><span class="meta">$</span> git clone https://github.com/jiangjiguang/iceberg.git</span><br><span class="line"><span class="meta">#</span> 将apache仓库添加为upstream</span><br><span class="line"><span class="meta">$</span> git remote add apache https://github.com/apache/iceberg.git</span><br><span class="line"><span class="meta">#</span> 从apache获取代码</span><br><span class="line"><span class="meta">$</span> git fetch apache master</span><br><span class="line"><span class="meta">#</span> 合并到本地分支</span><br><span class="line"><span class="meta">$</span> git merge apache/master</span><br><span class="line"><span class="meta">#</span> 推送到自己的仓库</span><br><span class="line"><span class="meta">$</span> git push origin master </span><br><span class="line"><span class="meta">#</span> 获取所有分支</span><br><span class="line"><span class="meta">$</span> git pull apache</span><br><span class="line"></span><br><span class="line"><span class="meta">#</span> 将所有标签推送到远程仓库</span><br><span class="line"><span class="meta">$</span> git push origin --tags</span><br></pre></td></tr></table></figure>


    </div>

    
      <footer class="post-footer">
        <div class="post-eof"></div>
      </footer>
  </article>
  
  
  <article itemscope itemtype="http://schema.org/Article" class="post-block home" lang="zh-CN">
    <link itemprop="mainEntityOfPage" href="http://yoursite.com/2021/03/18/数据湖/Hudi源码解读/">

    <span hidden itemprop="author" itemscope itemtype="http://schema.org/Person">
      <meta itemprop="name" content="姜纪光">
      <meta itemprop="description" content="">
      <meta itemprop="image" content="/images/avatar.gif">
    </span>

    <span hidden itemprop="publisher" itemscope itemtype="http://schema.org/Organization">
      <meta itemprop="name" content="纪光的博客">
    </span>
      <header class="post-header">
        <h1 class="post-title" itemprop="name headline">
          
            
            <a href="/2021/03/18/数据湖/Hudi源码解读/" class="post-title-link" itemprop="url">Hudi源码解读</a>
          
        </h1>

        <div class="post-meta">
            <span class="post-meta-item">
              <span class="post-meta-item-icon">
                <i class="fa fa-calendar-o"></i>
              </span>
              <span class="post-meta-item-text">发表于</span>

              
              <time title="创建时间：2021-03-18 16:20:19" itemprop="dateCreated datePublished" datetime="2021-03-18T16:20:19+08:00">2021-03-18</time>
            </span>
          
            
              <span class="post-meta-item">
                <span class="post-meta-item-icon">
                  <i class="fa fa-calendar-check-o"></i>
                </span>
                <span class="post-meta-item-text">更新于</span>
                <time title="修改时间：2021-06-20 12:19:35" itemprop="dateModified" datetime="2021-06-20T12:19:35+08:00">2021-06-20</time>
              </span>
            
          
            <span class="post-meta-item">
              <span class="post-meta-item-icon">
                <i class="fa fa-folder-o"></i>
              </span>
              <span class="post-meta-item-text">分类于</span>
              
                <span itemprop="about" itemscope itemtype="http://schema.org/Thing">
                  <a href="/categories/hudi/" itemprop="url" rel="index">
                    <span itemprop="name">hudi</span>
                  </a>
                </span>

                
            </span>
          

            <span id="/2021/03/18/数据湖/Hudi源码解读/" class="post-meta-item leancloud_visitors" data-flag-title="Hudi源码解读" title="阅读次数">
              <span class="post-meta-item-icon">
                <i class="fa fa-eye"></i>
              </span>
              <span class="post-meta-item-text">阅读次数：</span>
              <span class="leancloud-visitors-count"></span>
            </span>
          
  
        </div>
      </header>

    
    <div class="post-body" itemprop="articleBody">

      
          <h1 id="知识点"><a href="#知识点" class="headerlink" title="知识点"></a>知识点</h1><ul>
<li>COW表对应commit，MOR表对应deltacommit</li>
</ul>
<figure class="highlight java"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">String commitType = cfg.tableType.equals(HoodieTableType.COPY_ON_WRITE.name()) ? HoodieTimeline.COMMIT_ACTION : HoodieTimeline.DELTA_COMMIT_ACTION;</span><br></pre></td></tr></table></figure>

<ul>
<li><h1 id="参考文档"><a href="#参考文档" class="headerlink" title="参考文档"></a>参考文档</h1></li>
</ul>

        
    </div>

    
      <footer class="post-footer">
        <div class="post-eof"></div>
      </footer>
  </article>
  
  
  </div>

  
  <nav class="pagination">
    <span class="page-number current">1</span><a class="page-number" href="/page/2/">2</a><span class="space">&hellip;</span><a class="page-number" href="/page/16/">16</a><a class="extend next" rel="next" href="/page/2/"><i class="fa fa-angle-right" aria-label="下一页"></i></a>
  </nav>


          </div>
          

        </div>
          
  
  <div class="toggle sidebar-toggle">
    <span class="toggle-line toggle-line-first"></span>
    <span class="toggle-line toggle-line-middle"></span>
    <span class="toggle-line toggle-line-last"></span>
  </div>

  <aside class="sidebar">
    <div class="sidebar-inner">

      <ul class="sidebar-nav motion-element">
        <li class="sidebar-nav-toc">
          文章目录
        </li>
        <li class="sidebar-nav-overview">
          站点概览
        </li>
      </ul>

      <!--noindex-->
      <div class="post-toc-wrap sidebar-panel">
      </div>
      <!--/noindex-->

      <div class="site-overview-wrap sidebar-panel">
        <div class="site-author motion-element" itemprop="author" itemscope itemtype="http://schema.org/Person">
  <p class="site-author-name" itemprop="name">姜纪光</p>
  <div class="site-description" itemprop="description"></div>
</div>
<div class="site-state-wrap motion-element">
  <nav class="site-state">
      <div class="site-state-item site-state-posts">
        
          <a href="/archives/">
        
          <span class="site-state-item-count">158</span>
          <span class="site-state-item-name">日志</span>
        </a>
      </div>
    
      
      <div class="site-state-item site-state-categories">
        
          
            <a href="/categories/">
          
        
        <span class="site-state-item-count">26</span>
        <span class="site-state-item-name">分类</span>
        </a>
      </div>
    
      
      <div class="site-state-item site-state-tags">
        
          
            <a href="/tags/">
          
        
        <span class="site-state-item-count">14</span>
        <span class="site-state-item-name">标签</span>
        </a>
      </div>
    
  </nav>
</div>


      </div>

    </div>
  </aside>
  <div id="sidebar-dimmer"></div>


      </div>
    </main>

    <footer class="footer">
      <div class="footer-inner">
        

<div class="copyright">
  &copy; 
  <span itemprop="copyrightYear">2021</span>
  <span class="with-love">
    <i class="fa fa-user"></i>
  </span>
  <span class="author" itemprop="copyrightHolder">姜纪光</span>
</div>
  <div class="powered-by">由 <a href="https://hexo.io" class="theme-link" rel="noopener" target="_blank">Hexo</a> 强力驱动 v3.9.0
  </div>
  <span class="post-meta-divider">|</span>
  <div class="theme-info">
      
    主题 – <a href="https://mist.theme-next.org" class="theme-link" rel="noopener" target="_blank">NexT.Mist</a> v7.4.1
  </div>

        
      </div>
    </footer>
  </div>

  
  <script src="/lib/anime.min.js?v=3.1.0"></script>
  <script src="/lib/velocity/velocity.min.js?v=1.2.1"></script>
  <script src="/lib/velocity/velocity.ui.min.js?v=1.2.1"></script>
<script src="/js/utils.js?v=7.4.1"></script><script src="/js/motion.js?v=7.4.1"></script>
<script src="/js/schemes/muse.js?v=7.4.1"></script>

<script src="/js/next-boot.js?v=7.4.1"></script>


<script>
NexT.utils.getScript('//unpkg.com/valine/dist/Valine.min.js', () => {
  var GUEST = ['nick', 'mail', 'link'];
  var guest = 'nick,mail,link';
  guest = guest.split(',').filter(item => {
    return GUEST.includes(item);
  });
  new Valine({
    el: '#comments',
    verify: false,
    notify: false,
    appId: 'MIQC4jJTknc0hjNXDAiIuQ21-9Nh9j0Va',
    appKey: 'vaiXpSQLLMglwYsVLOBLzck4',
    placeholder: "Just go go",
    avatar: 'mm',
    meta: guest,
    pageSize: '10' || 10,
    visitor: true,
    lang: 'zh-cn' || 'zh-cn',
    path: location.pathname,
    recordIP: false,
    serverURLs: ''
  });
}, window.Valine);
</script>

</body>
</html>